## utils/hs_cleanup/hs_cleanup.c

/* [<][>][^][v][top][bottom][index][help] */

## FUNCTIONS

This source file includes following functions.

1. archive_history
2. archive_last
3. delete_history_entry
4. delete_last_entry
5. update_prev_serial_of_next_object
6. update_serial_in_last
7. update_serial_in_history
8. main
9. create_archive_tables
10. create_auxiliary_table
11. create_table
12. find_unreferenced_history_entries
13. delete_unreferenced_history_entries
14. PushTblObjList
15. get_highest_serial_before_date
16. archive_serials_and_history
17. lock_last_history_serial_tables
18. unlock_all_tables
19. update_hs_auxiliary_checkpoint
20. archive_serial
21. archive_failed_transaction
22. copy_into_history_archive
23. copy_deleted_object_into_history_archive
24. update_history_sequence_id_and_timestamp
25. check_if_next_is_deletion
26. update_prev_serial_of_object
27. update_serial_of_object
28. delete_serial_entry
29. delete_failed_transaction_entry
30. delete_entry_from_object_table
31. fetch_timestamp_from_last
32. optimize_sql_table
33. execute_sql_query
34. execute_sql_command
35. usage

       1 /***************************************
       2   $Revision: 1.22 $
       3
       4   History/Serial Cleanup (hs_cleanup). This utility archives serials
       5   and history entries, and deletes them from the live database.
       6
       7   Status: NOT COMPLETE, NOT REVUED, NOT FULLY TESTED
       8
       9   ******************/ /******************
      10   Filename            : hs_cleanup.c
      11   Authors             : daniele@ripe.net
      12   OSs Tested          : Solaris 7
      13   ******************/ /******************
      14   Copyright (c) 2000                              RIPE NCC
      15
      16   All Rights Reserved
      17
      18   Permission to use, copy, modify, and distribute this software and its
      19   documentation for any purpose and without fee is hereby granted,
      20   provided that the above copyright notice appear in all copies and that
      21   both that copyright notice and this permission notice appear in
      22   supporting documentation, and that the name of the author not be
      23   used in advertising or publicity pertaining to distribution of the
      24   software without specific, written prior permission.
      25
      26   THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
      27   ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
      28   AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
      29   DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
      30   AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
      31   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
      32   ***************************************/
      33
      34 #include <stdio.h>
      35 #include <stdlib.h>
      36 #include <string.h>
      37 #include <sys/time.h>
      38
      39 #include "mysql_driver.h"
      40 #include "stubs.h"
      41
      42
      43
      44 #define MYSQL_HOST "myhost.mydb.net"
      45 #define MYSQL_PORT 3306
      46 #define MYSQL_USER "sqluser"
      47 #define MYSQL_PSWD "sqlpswd"
      48 #define MYSQL_DB "sqldb"
      49
      50 /* String sizes */
      51 #define STR_S   63
      52 #define STR_M   255
      53 #define STR_L   1023
      54 #define STR_XL  4095
      55 #define STR_XXL 16383
      56
      57 #define MAXCMDLEN 8192
      58 #define MAXNUMCMDS 65536
      59
      60 #define DATE 966050936
      61
      62 /* Set the default lapse before which we want to archive:
      63  * it must be in seconds.
      64  * 1min = 60s
      65  * 1hr = 3600s
      66  * 1day = 86400s
      67  * 1wk = 604800s
      68  */
      69
      70 #define MINUTE 60
      71 #define HOUR 3600
      72 #define DAY 86400
      73 #define WEEK 604800
      74
      75 #define DEBUG 1
      76 #define ARCHIVE_ONLY 1
      77
      78
      79 enum {
      80   IN_HISTORY_TABLE = 0,
      81   IN_LAST_TABLE,
      82   IN_FAILED_TRANSACTION_TABLE,
      83   IN_UNDEF_TABLE
      84 };
      85
      86 enum {
      87   OP_NULL = 0,
      88   OP_ADD,
      89   OP_DELETE,
      90   OP_UPDATE
      91 };
      92
      93
      94 enum {
      95   CHKP_NOOP = 1,
      96   CHKP_ARCHIVED_SERIAL,
      97   CHKP_ARCHIVED_HISTORY,
      98   CHKP_DELETED_HISTORY,
      99   CHKP_DELETED_SERIAL,
     100   CHKP_DONE
     101 };
     102
     103 typedef struct table_object *tblobjPtr;
     104
     105 typedef struct table_object {
     106   int objid;
     107   int seqid;
     108   tblobjPtr next;
     109 } tblObjList;
     110
     111
     112 /* Global variables */
     113
     114 SQ_connection_t *connection;
     115 int debug = DEBUG;
     116 int archive_only = ARCHIVE_ONLY;
     117
     118 /* Function definitions */
     119
     120 int main (int argc, char *argv[]);
     121 void usage(char *argv[]);
     122
     123 static tblObjList *find_unreferenced_history_entries(int date);
     124 static int delete_unreferenced_history_entries(tblObjList *objectsToDelete);
     125 static int create_auxiliary_table();
     126 static int create_archive_tables();
     127 static int create_table(const char *cmd);
     128 static int get_highest_serial_before_date(int *highest_serial_ptr, int date);
     129 static int archive_serials_and_history(int highest_serial);
     130 static int archive_serial(int ser_id, int obj_id, int seq_id, int op);
     131 static int archive_failed_transaction(int ser_id);
     132 static int copy_into_history_archive(int ser_id, int obj_id, int seq_id, const char *tablename);
     133 static int copy_deleted_object_into_history_archive (int ser_id, int obj_id, int seq_id, const char *tablename);
     134 static int update_history_sequence_id_and_timestamp(int ser_id, int new_seq_id, int new_timestamp);
     135 static int delete_serial_entry(int ser_id);
     136 static int delete_failed_transaction_entry (int ser_id);
     137 static int delete_entry_from_object_table(int obj_id, int seq_id, const char* tablename);
     138 static int fetch_timestamp_from_last(int obj_id, int ser_id);
     139 static int execute_sql_command(const char *cmd);
     140 static int execute_sql_query(const char *query, SQ_result_set_t **result_ptr);
     141 static int check_if_next_is_deletion (int obj_id, int seq_id);
     142 static int update_prev_serial_of_object (int obj_id, int seq_id, int prev_ser, const char *tablename);
     143 static int update_serial_of_object (int obj_id, int seq_id, int ser_id, const char *tablename);
     144 static int lock_last_history_serial_tables();
     145 static int unlock_all_tables();
     146 static int optimize_sql_table(const char* tablename);
     147 static int update_hs_auxiliary_checkpoint(int ser_id, int obj_id, int seq_id, int timestamp, int checkpoint);
     148
     149
     150
     151 #define archive_history(ser_id, obj_id, seq_id) copy_into_history_archive(ser_id, obj_id, seq_id, "history")
         /* [<][>][^][v][top][bottom][index][help] */
     152 #define archive_last(ser_id, obj_id, seq_id) copy_into_history_archive(ser_id, obj_id, seq_id, "last")
         /* [<][>][^][v][top][bottom][index][help] */
     153 #define delete_history_entry(obj_id, seq_id) delete_entry_from_object_table(obj_id, seq_id, "history")
         /* [<][>][^][v][top][bottom][index][help] */
     154 #define delete_last_entry(obj_id, seq_id) delete_entry_from_object_table(obj_id, seq_id, "last")
         /* [<][>][^][v][top][bottom][index][help] */
     155 #define update_prev_serial_of_next_object(obj_id, seq_id, prev_ser, tablename) update_prev_serial_of_object(obj_id, seq_id+1, prev_ser, tablename)
         /* [<][>][^][v][top][bottom][index][help] */
     156 #define update_serial_in_last(obj_id, seq_id, ser_id) update_serial_of_object(obj_id, seq_id, ser_id, "last")
         /* [<][>][^][v][top][bottom][index][help] */
     157 #define update_serial_in_history(obj_id, seq_id, ser_id) update_serial_of_object(obj_id, seq_id, ser_id, "history")
         /* [<][>][^][v][top][bottom][index][help] */
     158
     159
     160
     161 /* XXX Fixme:
     162    - No more hardcoded variables
     163    - Subroutine (+option) to warn that the size is bigger than a watermark
     164    - Checkpointing for crash recovery
     165    */
     166
     167
     168
     169 int main (int argc, char *argv[])
         /* [<][>][^][v][top][bottom][index][help] */
     170 {
     171
     172   int ch, rc;
     173   int highest_serial;
     174   short errflg = 1;
     175   short tflg = 0;
     176   extern char *optarg;
     177   extern int optind;
     178   time_t now = time(0);
     179   time_t date;
     180   time_t lapse;
     181
     182   char sqlhost[STR_M] = MYSQL_HOST;
     183   int sqlport = MYSQL_PORT;
     184   char sqluser[STR_M] = MYSQL_USER;
     185   char sqlpswd[STR_M] = MYSQL_PSWD;
     186   char sqldb[STR_M] = MYSQL_DB;
     187
     188   tblObjList *objectsToDelete;
     189   tblObjList *tmpObj;
     190
     191   /* Get options */
     192
     193   while ((ch = getopt(argc, argv, "?T:S:M:H:D:W:h:P:u:p:d:")) != EOF )
     194     switch((char)ch)
     195       {
     196       case 'T':
     197         if (tflg)
     198           errflg++;
     199         else
     200           {
     201             tflg++;
     202             errflg = 0;
     203             date = atol (optarg);
     204           }
     205         break;
     206       case 'S':
     207         if (tflg)
     208           errflg++;
     209         else
     210           {
     211             tflg++;
     212             errflg = 0;
     213             lapse = atol (optarg);
     214             date = now - lapse;
     215           }
     216         break;
     217       case 'M':
     218         if (tflg)
     219           errflg++;
     220         else
     221           {
     222             tflg++;
     223             errflg = 0;
     224             lapse = atol (optarg);
     225             date = now - lapse * MINUTE;
     226           }
     227         break;
     228       case 'H':
     229         if (tflg)
     230           errflg++;
     231         else
     232           {
     233             tflg++;
     234             errflg = 0;
     235             lapse = atol (optarg);
     236             date = now - lapse * HOUR;
     237           }
     238         break;
     239       case 'D':
     240         if (tflg)
     241           errflg++;
     242         else
     243           {
     244             tflg++;
     245             errflg = 0;
     246             lapse = atol (optarg);
     247             date = now - lapse * DAY;
     248           }
     249         break;
     250       case 'W':
     251         if (tflg)
     252           errflg++;
     253         else
     254           {
     255             tflg++;
     256             errflg = 0;
     257             lapse = atol (optarg);
     258             date = now - lapse * WEEK;
     259           }
     260         break;
     261       case 'h':
     262         sprintf (sqlhost,"%s",optarg);
     263         break;
     264       case 'P':
     265         sqlport = atoi(optarg);
     266         break;
     267       case 'u':
     268         sprintf (sqluser,"%s",optarg);
     269         break;
     270       case 'p':
     271         sprintf (sqlpswd,"%s",optarg);
     272         break;
     273       case 'd':
     274         sprintf (sqldb,"%s",optarg);
     275         break;
     276       case '?':
     277       default:
     278         errflg++;
     279       }
     280
     281 if (errflg)
     282   usage(argv);
     283
     284
     285   /* Initialize connection */
     286   connection = SQ_get_connection(sqlhost, sqlport,sqldb,sqluser,sqlpswd);
     287
     288   /* Create tables for history and serials archives
     289    * if they do not exist */
     290   if ((rc = create_archive_tables()) != 0)
     291     { return(rc); }
     292
     293   /* Create auxiliary table if it does not exist */
     294   if ((rc = create_auxiliary_table()) != 0)
     295     { return(rc); }
     296
     297   /* XXX Call remadmin interface and stop updates */
     298
     299
     300   /* XXX If retcode is successful, go on */
     301
     302   /* Deal with very old history entries, those that do not even have a corresponding
     303    * serial. These are entries which had been archived when they were in the "last" table,
     304    * and have in the meanwhile been updated. We have to:
     305    *    - Update prev_serial of their next object
     306    *    - Delete them!
     307    */
     308
     309   /* XXX Not fully tested: some code must be updated in UD to support the extra columns
     310      serial and prev_serial */
     311   objectsToDelete = find_unreferenced_history_entries((int) date);
     312
     313   /* printf ("Elements to be deleted:\n");
     314      for (tmpObj = objectsToDelete; tmpObj != NULL; tmpObj = tmpObj->next)
     315      {
     316      printf ("objid: %d, seqid: %d\n", tmpObj->objid, tmpObj->seqid);
     317      } */
     318
     319
     320   /* Get the biggest serial for which the history or last timestamp is lower than
     321    * the defined timestamp
     322    */
     323
     324   if ((rc = get_highest_serial_before_date(&highest_serial, (int) date)) != 0)
     325     { return(rc); }
     326   printf ("Highest serial ID: %d\n",highest_serial);
     327
     328
     329   /* Execute the archiving commands */
     330
     331   archive_serials_and_history(highest_serial);
     332
     333   /* Optimize history serial and last tables: there might have been many deletions */
     334   optimize_sql_table("serials");
     335   optimize_sql_table("history");
     336   optimize_sql_table("last");
     337
     338   /* XXX Call remadmin interface and restart updates */
     339
     340
     341   /* XXX If retcode is not successful, go on, issue a warning */
     342
     343   /* Delete the unreferenced history entries. Must be done at the end. */
     344   /* XXX Bug here. The older entries cannot be deleted or they wreak havoc. */
     345   if (! archive_only)
     346     delete_unreferenced_history_entries(objectsToDelete);
     347
     348   /* OK, it's over. */
     349
     350   SQ_close_connection(connection);
     351   printf ("\nProgram done.\n");
     352   return(0);
     353
     354 } /* main() */
     355
     356
     357
     358
     359
     360 /* Functions */
     361
     362
     363 /* create_archive_tables():
     364  * Create tables for history and serials archives
     365  * if they do not exist */
     366
     367 int create_archive_tables()
         /* [<][>][^][v][top][bottom][index][help] */
     368 {
     369
     370   char cmd[MAXCMDLEN];
     371
     372   sprintf (cmd,
     373            "CREATE TABLE history_archive (
     374                 object_id int(10) unsigned DEFAULT '0' NOT NULL,
     375                 sequence_id int(10) unsigned DEFAULT '0' NOT NULL,
     376                 serial int(11) DEFAULT '0' NOT NULL,
     377                 prev_serial int(11) DEFAULT '0' NOT NULL,
     378                 timestamp int(10) unsigned DEFAULT '0' NOT NULL,
     379                 object_type tinyint(3) unsigned DEFAULT '0' NOT NULL,
     380                 object longblob NOT NULL,
     381                 PRIMARY KEY (object_id,sequence_id,serial)
     382                 );");
     383   create_table(cmd);
     384
     385
     386   sprintf (cmd,
     387            "CREATE TABLE serials_archive (
     388                 serial_id int(11) DEFAULT '0' NOT NULL,
     389                 object_id int(10) unsigned DEFAULT '0' NOT NULL,
     390                 sequence_id int(10) unsigned DEFAULT '0' NOT NULL,
     391                 operation tinyint(4) unsigned DEFAULT '0' NOT NULL,
     392                 PRIMARY KEY (serial_id)
     393                 );");
     394   create_table(cmd);
     395
     396   return(0);
     397
     398 } /* create_archive_tables() */
     399
     400
     401
     402 /* create_auxiliary_table()
     403  * This auxiliary table will record some checkpointing
     404  * data, in order to recover from crashes
     405  * and to help with the clenup of the older history tables
     406  */
     407 int create_auxiliary_table()
         /* [<][>][^][v][top][bottom][index][help] */
     408 {
     409
     410   int state;
     411   char cmd[MAXCMDLEN];
     412
     413   sprintf (cmd,"CREATE TABLE hs_auxiliary (
     414                 serial int(11) DEFAULT '0' NOT NULL,
     415                 object_id int(10) unsigned DEFAULT '0' NOT NULL,
     416                 sequence_id int(10) unsigned DEFAULT '0' NOT NULL,
     417                 timestamp int(10) unsigned DEFAULT '0' NOT NULL,
     418                 checkpoint tinyint(4) unsigned DEFAULT '0' NOT NULL,
     419                 PRIMARY KEY (serial)
     420                 );");
     421   state = create_table(cmd);
     422   if (state == 0) /* state != 0 only if the table already exists - other errors make the program die */
     423     {
     424       /* We also need to create a dummy row if the table had not been created */
     425       sprintf (cmd,"INSERT INTO hs_auxiliary VALUES (0, 0, 0, 0, 0)");
     426       execute_sql_command(cmd);
     427     }
     428
     429   return(0);
     430
     431 } /* create_auxiliary_table() */
     432
     433
     434
     435
     436 /* create_table()
     437  * This function wraps a table creation (which must be already
     438  * specified in cmd), and only issues a warning if the table
     439  * already exists.
     440  */
     441
     442 int create_table (const char *cmd)
         /* [<][>][^][v][top][bottom][index][help] */
     443 {
     444
     445   int state;
     446
     447   state = SQ_execute_query(connection, cmd, NULL);
     448   if (state != 0)
     449     {
     450       /* XXX is ER_TABLE_EXISTS_ERROR mysql-bounded? */
     451       if (SQ_errno(connection) == ER_TABLE_EXISTS_ERROR)
     452         {
     453           /* Don't die if a table already exists */
     454           fprintf (stderr,"Warning: %s\n",SQ_error(connection));
     455           return (state);
     456         }
     457       else
     458         {
     459           fprintf (stderr,"Fatal: %s\n",SQ_error(connection));
     460           dieif (state != 0);
     461         }
     462     }
     463
     464   return(0);
     465
     466 } /* create_table() */
     467
     468
     469
     470
     471 /* find_unreferenced_history_entries()
     472  * Deal with very old history entries, those that are not referenced by any serial,
     473  * due to a previous history/serial cleanup.
     474  * These are entries which had been archived when they were in the "last" table,
     475  * and have in the meanwhile been updated. We have to:
     476  *    - Update prev_serial of their next object
     477  *    - Delete them!
     478  */
     479
     480 tblObjList *find_unreferenced_history_entries(int date)
         /* [<][>][^][v][top][bottom][index][help] */
     481 {
     482
     483   char query[MAXCMDLEN];
     484   SQ_result_set_t *result;
     485   SQ_row_t *row;
     486   int objid, seqid, serid;
     487
     488   tblObjList *curListElmt = NULL;
     489   tblObjList *firstListElmt = NULL;
     490   tblObjList *tmpList = NULL;
     491
     492   /* Find object_id, sequence_id of unreferenced history entries
     493    * This query returns all history entries which do not have a corresponding
     494    * (object_id, serial_id) in the serials table
     495    */
     496   /* XXX Bug! This will find (and then remove) objects that would be
     497    * needed in the future by deletions. */
     498
     499   sprintf (query, "SELECT history.object_id, history.sequence_id, history.serial
     500                         FROM history LEFT JOIN serials
     501                         ON history.serial = serials.serial_id
     502                         WHERE serials.serial_id is NULL
     503                         AND history.serial != 0
     504                         AND history.timestamp < %d", date);
     505   execute_sql_query(query, &result);
     506
     507   /* Foreach entry: */
     508   while ( (row = SQ_row_next(result)) != NULL )
     509     {
     510
     511       /* Lock tables in writing... */
     512       /* XXX We don't need to do it, we are not deleting the objects here! */
     513       /* lock_last_history_serial_tables(); */
     514
     515       /* XXX Error checking missing... */
     516       objid = atoi((const char *)row[0]);
     517       seqid = atoi((const char *)row[1]);
     518       serid = atoi((const char *)row[2]);
     519
     520       /* Update prev_serial of the same object with next sequence_id */
     521       if (!update_prev_serial_of_next_object(objid, seqid, serid, "history"))
     522         { update_prev_serial_of_next_object(objid, seqid, serid, "last"); }
     523
     524       /* Delete the entry */
     525       printf ("I am deleting this entry: %d, %d\n",objid, seqid);
     526
     527       /* Don't delete the history entries directly! This will cause problems
     528          if the next serial is a deletion */
     529       /* Instead, add it in a list that will be batch-deleted at the end */
     530       /* PushTblObjList (objid, seqid, curListElmt); */
     531
     532       tmpList = (tblObjList *)malloc(sizeof(tblObjList));
     533       tmpList->objid = objid;
     534       tmpList->seqid = seqid;
     535       tmpList->next = NULL;
     536
     537       if (firstListElmt == NULL)
     538         {
     539           firstListElmt = tmpList;
     540           curListElmt = tmpList;
     541         }
     542       else
     543         {
     544           curListElmt->next = tmpList;
     545           curListElmt = curListElmt->next;
     546         }
     547
     548       /* Unlock tables... */
     549       /* unlock_all_tables(); */
     550
     551     }
     552
     553   /* printf ("Elements to be deleted:\n");
     554      for (curListElmt = firstListElmt; curListElmt != NULL; curListElmt = curListElmt->next)
     555      {
     556      printf ("objid: %d, seqid: %d\n", curListElmt->objid, curListElmt->seqid);
     557      } */
     558
     559   return (firstListElmt);
     560
     561   return (0);
     562
     563 } /* find_unreferenced_history_entries() */
     564
     565
     566 int delete_unreferenced_history_entries(tblObjList *objectsToDelete)
         /* [<][>][^][v][top][bottom][index][help] */
     567 {
     568
     569   tblObjList *tmpObj;
     570
     571   printf ("Elements to be deleted:\n");
     572   for (tmpObj = objectsToDelete; tmpObj != NULL; tmpObj = tmpObj->next)
     573     {
     574       printf ("objid: %d, seqid: %d\n", tmpObj->objid, tmpObj->seqid);
     575       delete_history_entry(tmpObj->objid, tmpObj->seqid);
     576     }
     577
     578   return(0);
     579
     580 }
     581
     582
     583 /* PushTblObjList() */
     584
     585 int PushTblObjList(int objid, int seqid, tblObjList *curListElmt)
         /* [<][>][^][v][top][bottom][index][help] */
     586 {
     587
     588   tblObjList *tmpList;
     589
     590   tmpList = (tblObjList *)malloc(sizeof(tblObjList));
     591   tmpList->objid = objid;
     592   tmpList->seqid = seqid;
     593   tmpList->next = NULL;
     594
     595   if (curListElmt == NULL)
     596     {
     597       curListElmt = tmpList;
     598     }
     599   else
     600     {
     601       curListElmt->next = tmpList;
     602       /* curListElmt = tmpList; */
     603     }
     604
     605   printf ("Inside PushTblObjList: %d, %d\n", curListElmt->objid, curListElmt->seqid);
     606
     607   return(0);
     608
     609 } /* PushTblObjList() */
     610
     611
     612 /* get_highest_serial_before_date()
     613  * We get the biggest serial for which the history or last timestamp is lower than
     614  * the defined timestamp
     615  */
     616
     617 int get_highest_serial_before_date (int *highest_serial_ptr, int date)
         /* [<][>][^][v][top][bottom][index][help] */
     618 {
     619
     620   char query[MAXCMDLEN];
     621   SQ_result_set_t *result;
     622   SQ_row_t *row;
     623
     624   /* sprintf (query, "SELECT MAX(serials.serial_id) FROM history,serials
     625                         WHERE history.timestamp < %d
     626                         AND history.object_id = serials.object_id
     627                         AND history.sequence_id = serials.sequence_id ", date); */
     628
     629   sprintf (query, "SELECT MAX(serials.serial_id)
     630                         FROM serials NATURAL LEFT JOIN last NATURAL LEFT JOIN history
     631                         WHERE ((last.timestamp < %d
     632                                 AND last.object_id = serials.object_id
     633                                 AND last.sequence_id = last.sequence_id)
     634                         OR (history.timestamp < %d
     635                                 AND history.object_id = serials.object_id
     636                                 AND history.sequence_id = serials.sequence_id))",
     637            date, date);
     638
     639   execute_sql_query(query, &result);
     640   if ( (row = SQ_row_next(result)) != NULL )
     641     {
     642       *highest_serial_ptr = row[0] ? atoi((const char *)row[0]) : 0;
     643       /* printf ("Highest serial ID: %d\n", *highest_serial_ptr); */
     644     }
     645
     646   SQ_free_result(result);
     647
     648   return(0);
     649
     650 } /* get_highest_serial_before_date() */
     651
     652
     653
     654
     655
     656 /* archive_serials_and_history():
     657  * This function contains the core algorithm that manipulates the last,
     658  * history and serials tables and archives them into serials_archive
     659  * and history_archive tables.
     660  */
     661
     662 int archive_serials_and_history (int highest_serial)
         /* [<][>][^][v][top][bottom][index][help] */
     663 {
     664
     665   char query[MAXCMDLEN];
     666   SQ_result_set_t *result;
     667   SQ_row_t *row;
     668   int serial, atlast, objid, seqid, op;
     669   char *tablename;
     670   int timestamp;
     671   int rc;
     672
     673
     674   /* XXX Missing: crash recovery handling. */
     675
     676
     677   /* Get the entries for each serial */
     678   /* One word about the "<": I had "<=" but if highest_serial
     679      is the CURRENTSERIAL, it causes big problems to UD!
     680      (at least in mirror mode...) */
     681   sprintf (query, "SELECT serials.serial_id, serials.atlast,
     682                         ELT(serials.atlast+1,'history','last','failed_transaction'),
     683                         serials.object_id, serials.sequence_id, serials.operation
     684                         FROM serials
     685                         WHERE serials.serial_id < %d
     686                         ORDER BY serials.serial_id", highest_serial);
     687   execute_sql_query(query, &result);
     688
     689   /* Loop on every serial */
     690   while ( (row = SQ_row_next(result)) != NULL )
     691     {
     692
     693       /* The lock is inserted here, inside the loop, because it is
     694        * a write lock, which disallows the reading of the table.
     695        * Since one concerned table is "last", the queries would
     696        * be blocked.
     697        * By freeing the lock at the end of every loop, we are assured
     698        * that the reads in queue are executed before a new lock is set.
     699        */
     700
     701       /* Lock (write lock!) relevant tables */
     702       if ((rc = lock_last_history_serial_tables()) != 0)
     703         { return rc; }
     704
     705       /* XXX Add stronger error checking: NULL rows should never happen */
     706       serial = row[0] ? atoi((const char *)row[0]) : 0;
     707       atlast = row[1] ? atoi((const char *)row[1]) : IN_UNDEF_TABLE;
     708
     709       if (row[2] == NULL)
     710         {
     711           /* That should never happen! */
     712           fprintf (stderr, "Fatal: No pointer to table\n");
     713           return (-1);
     714         }
     715       else
     716         {
     717           tablename = strdup((const char *)row[2]);
     718         }
     719
     720       objid = atoi((const char *)row[3]);
     721       seqid = atoi((const char *)row[4]);
     722       op = atoi((const char *)row[5]);
     723
     724       /* printf ("Serial: %d; Atlast: %d; Objid: %d; Seqid: %d; Op: %d; Tablename: %s\n",serial, atlast, objid, seqid, op, tablename); */
     725
     726       update_hs_auxiliary_checkpoint(serial, objid, seqid, 0, CHKP_NOOP);
     727
     728       if (atlast == IN_FAILED_TRANSACTION_TABLE)
     729         {
     730
     731           /* The serial points to a failed transaction */
     732
     733           /* Archive serial */
     734           archive_serial(serial, objid, seqid, op);
     735           update_hs_auxiliary_checkpoint(serial, objid, seqid, 0, CHKP_ARCHIVED_SERIAL);
     736
     737           /* Archive failed transaction */
     738           archive_failed_transaction(serial);
     739
     740           /* Delete serial */
     741           delete_serial_entry(serial);
     742
     743           /* Delete failed transaction */
     744           delete_failed_transaction_entry(serial);
     745
     746
     747         }
     748       else /* atlast == (IN_LAST_TABLE || IN_HISTORY_TABLE) */
     749         {
     750
     751           if (op == OP_DELETE)
     752             {
     753
     754               /* Then it must be in the history */
     755
     756               if (debug) printf ("Deleted serial. Objid: %d, seqid: %d, serial: %d\n",objid, seqid, serial);
     757
     758               /* We need to update the prev_serial of the next element, if there is one...
     759                * This compensates for UPD = DEL + ADD; the ADD is treated with the same
     760                * object_id and sequence_id++ */
     761               if (!update_prev_serial_of_next_object(objid, seqid, serial, "history"))
     762                 update_prev_serial_of_next_object(objid, seqid, serial, "last");
     763
     764               /* Archive serial */
     765               archive_serial(serial, objid, seqid, op);
     766
     767               /* XXX Fixme: no timestamp is archived if this DEL is part of a DEL+ADD .
     768                * This could be solved by fetching the timestamp from the next
     769                * sequence_id (which is the corresponding UPD), if existent.
     770                */
     771
     772               /* Fetch timestamp from the corresponding empty last entry */
     773               timestamp = fetch_timestamp_from_last(objid, seqid);
     774
     775               /* printf ("Timestamp for serial %d: %d\n",serial, timestamp); */
     776
     777               /* Archive history:
     778                * we need a special function here because we need to archive
     779                * history.serial as history_archive.prev_serial .
     780                */
     781               copy_deleted_object_into_history_archive(serial, objid, seqid, "history");
     782
     783               /* Update history archive with correct timestamp */
     784               /* XXX We don't really need a function which also updates the seq_id */
     785               update_history_sequence_id_and_timestamp(serial, seqid, timestamp);
     786
     787
     788               /* Delete serial */
     789               delete_serial_entry(serial);
     790
     791               /* Delete corresponding empty last entry: it has a seq_id of 0 */
     792               /* XXX It must only do so if the entry to be deleted is not the
     793                  highest object_id */
     794               /* To be fixed */
     795               /* delete_last_entry(objid, 0); */
     796
     797               /* Delete history entry */
     798               delete_history_entry(objid, seqid);
     799
     800
     801             }
     802           else /* It is an update */
     803             {
     804
     805               if (atlast == IN_LAST_TABLE )
     806                 {
     807
     808                   /* Archive serial */
     809                   archive_serial(serial, objid, seqid, op);
     810
     811                   /* Archive last */
     812                   archive_last(serial, objid, seqid);
     813
     814                   /* Update serial element of the entry in last table */
     815                   update_serial_in_last(objid, seqid, serial);
     816
     817                   /* Delete serial */
     818                   delete_serial_entry(serial);
     819
     820                   /* !!!Do not delete the "last" entry!!! */
     821
     822                 }
     823               else /* atlast == IN_HISTORY_TABLE */
     824                 {
     825
     826                   /* We check for the next object, in order to update
     827                    * its prev_serial. We first look in the history table,
     828                    * then in the last table, otherwise there is no such object
     829                    * => the following update is in fact a deletion...
     830                    */
     831
     832                   if (check_if_next_is_deletion == 0)
     833                     {
     834
     835                       /* update_prev_serial_of_next_object() returns the number of
     836                        * affected rows: this shows us if the operation has been successful
     837                        * or not */
     838                       if (!update_prev_serial_of_next_object(objid, seqid, serial, "history"))
     839                         update_prev_serial_of_next_object(objid, seqid, serial, "last");
     840
     841                       /* Archive serial */
     842                       archive_serial(serial, objid, seqid, op);
     843
     844                       /* Archive history */
     845                       archive_history(serial, objid, seqid);
     846
     847                       /* Delete serial */
     848                       delete_serial_entry(serial);
     849
     850                       /* Delete history */
     851                       delete_history_entry(objid, seqid);
     852
     853                     }
     854                   else
     855                     {
     856
     857                       /* Archive serial */
     858                       archive_serial(serial, objid, seqid, op);
     859
     860                       /* Archive history */
     861                       archive_history(serial, objid, seqid);
     862
     863                       /* Update serial in -current- history entry */
     864                       update_serial_in_history(objid, seqid, serial);
     865
     866                       /* Delete serial */
     867                       delete_serial_entry(serial);
     868
     869                       /* Do not delete current history entry! It will be needed
     870                          by the deleting serial */
     871
     872                     }
     873
     874                 }
     875
     876             }
     877
     878         }
     879
     880       /* Unlock relevant tables */
     881       unlock_all_tables();
     882
     883     }
     884
     885   SQ_free_result(result);
     886
     887
     888
     889   return(0);
     890
     891 } /* archive_serials_and_history() */
     892
     893
     894
     895
     896
     897
     898 int lock_last_history_serial_tables()
         /* [<][>][^][v][top][bottom][index][help] */
     899 {
     900
     901   char cmd[MAXCMDLEN];
     902
     903   /* No real choice - we must lock the tables in write mode */
     904
     905   sprintf (cmd, "LOCK TABLES last WRITE, history WRITE, failed_transaction WRITE, serials WRITE, serials_archive WRITE, history_archive WRITE, hs_auxiliary WRITE");
     906
     907   return (execute_sql_command(cmd));
     908
     909 } /* lock_last_history_serial_tables() */
     910
     911
     912
     913 int unlock_all_tables()
         /* [<][>][^][v][top][bottom][index][help] */
     914 {
     915
     916   char cmd[MAXCMDLEN];
     917
     918   sprintf (cmd, "UNLOCK TABLES");
     919
     920   return (execute_sql_command(cmd));
     921
     922 } /* unlock_all_tables() */
     923
     924
     925 int update_hs_auxiliary_checkpoint(int ser_id, int obj_id, int seq_id, int timestamp, int checkpoint)
         /* [<][>][^][v][top][bottom][index][help] */
     926 {
     927
     928   char cmd[MAXCMDLEN];
     929
     930   sprintf (cmd,"UPDATE hs_auxiliary
     931                 SET serial = %d,
     932                 object_id = %d,
     933                 sequence_id = %d,
     934                 timestamp = %d,
     935                 checkpoint = %d",
     936            ser_id, obj_id, seq_id, timestamp, checkpoint);
     937
     938   return (execute_sql_command(cmd));
     939
     940 } /* update_hs_auxiliary_checkpoint() */
     941
     942
     943 int archive_serial (int ser_id, int obj_id, int seq_id, int op)
         /* [<][>][^][v][top][bottom][index][help] */
     944 {
     945
     946   /* Put the given values into the serials_archive table */
     947
     948   char cmd[MAXCMDLEN];
     949
     950   sprintf (cmd, "INSERT INTO serials_archive (serial_id, object_id, sequence_id, operation)
     951                  VALUES (%d, %d, %d, %d)",
     952            ser_id, obj_id, seq_id, op);
     953
     954   return (execute_sql_command(cmd));
     955
     956 } /* archive_serial() */
     957
     958
     959
     960
     961 int archive_failed_transaction (int ser_id)
         /* [<][>][^][v][top][bottom][index][help] */
     962 {
     963
     964   char cmd[MAXCMDLEN];
     965
     966   sprintf (cmd,"INSERT INTO history_archive (serial, timestamp, object)
     967                 SELECT failed_transaction.serial_id, failed_transaction.timestamp, failed_transaction.object
     968                 FROM failed_transaction
     969                 WHERE failed_transaction.serial_id = %d",
     970            ser_id);
     971
     972   return (execute_sql_command(cmd));
     973
     974 } /* archive_failed_transaction() */
     975
     976
     977 int copy_into_history_archive (int ser_id, int obj_id, int seq_id, const char *tablename)
         /* [<][>][^][v][top][bottom][index][help] */
     978 {
     979
     980   char cmd[MAXCMDLEN];
     981
     982   sprintf (cmd,"INSERT INTO history_archive (object_id, sequence_id, serial, prev_serial, timestamp, object_type, object)
     983                 SELECT serials.object_id, serials.sequence_id, serials.serial_id, %s.prev_serial, %s.timestamp, %s.object_type, %s.object
     984                 FROM serials,%s
     985                 WHERE serials.serial_id = %d
     986                 AND %s.object_id = %d
     987                 AND %s.sequence_id = %d",
     988            tablename, tablename, tablename, tablename, tablename, ser_id, tablename, obj_id, tablename, seq_id);
     989
     990   return (execute_sql_command(cmd));
     991
     992 } /* copy_into_history_archive() */
     993
     994
     995
     996 int copy_deleted_object_into_history_archive (int ser_id, int obj_id, int seq_id, const char *tablename)
         /* [<][>][^][v][top][bottom][index][help] */
     997 {
     998
     999   /* The difference here is that we archive the history.serial as history_archive.prev_serial .
    1000    * This is only needed for history objects corresponding to OP_DELETE,
    1001    * where the row actually corresponds to the last update before the deletion. */
    1002
    1003   char cmd[MAXCMDLEN];
    1004   int affected_rows;
    1005
    1006   sprintf (cmd,"INSERT INTO history_archive (object_id, sequence_id, serial, prev_serial, timestamp, object_type, object)
    1007                 SELECT serials.object_id, serials.sequence_id, serials.serial_id, %s.serial, %s.timestamp, %s.object_type, %s.object
    1008                 FROM serials,%s
    1009                 WHERE serials.serial_id = %d
    1010                 AND %s.object_id = %d
    1011                 AND %s.sequence_id = %d",
    1012            tablename, tablename, tablename, tablename, tablename, ser_id, tablename, obj_id, tablename, seq_id);
    1013
    1014   affected_rows = execute_sql_command(cmd);
    1015   if (debug) printf ("copy_deleted_object_into_history_archive (%d, %d, %d, %s): affected rows %d\n",ser_id,obj_id,seq_id,tablename,affected_rows);
    1016   return (affected_rows);
    1017
    1018 } /* copy_deleted_object_into_history_archive() */
    1019
    1020
    1021
    1022 int update_history_sequence_id_and_timestamp (int ser_id, int new_seq_id, int new_timestamp)
         /* [<][>][^][v][top][bottom][index][help] */
    1023 {
    1024
    1025   char cmd[MAXCMDLEN];
    1026
    1027   sprintf (cmd,"UPDATE history_archive
    1028                 SET timestamp = %d, sequence_id = %d
    1029                 WHERE serial = %d",
    1030            new_timestamp, new_seq_id, ser_id);
    1031
    1032   return (execute_sql_command(cmd));
    1033
    1034 } /* update_history_sequence_id_and_timestamp() */
    1035
    1036
    1037
    1038 /* check_if_next_is_deletion()
    1039  * This functions checks if there is a row in the serials
    1040  * table with same obj_id and seq_id, but a delete operation.
    1041  * This would mean that we are dealing with a last-update-before-deletion.
    1042  */
    1043
    1044 int check_if_next_is_deletion (int obj_id, int seq_id)
         /* [<][>][^][v][top][bottom][index][help] */
    1045 {
    1046
    1047   char query[MAXCMDLEN];
    1048   SQ_result_set_t *result;
    1049   SQ_row_t *row;
    1050   int serial = 0;
    1051
    1052   sprintf (query, "SELECT serial_id, atlast FROM serials
    1053                         WHERE object_id = %d AND sequence_id = %d AND operation = %d",
    1054            obj_id, seq_id, OP_DELETE);
    1055
    1056   execute_sql_query(query, &result);
    1057   if ( (row = SQ_row_next(result)) != NULL)
    1058     serial = atoi((const char *)row[0]);
    1059
    1060   SQ_free_result(result);
    1061
    1062   return(serial);
    1063
    1064 } /* check_if_next_is_deletion() */
    1065
    1066
    1067
    1068 int update_prev_serial_of_object (int obj_id, int seq_id, int prev_ser, const char *tablename)
         /* [<][>][^][v][top][bottom][index][help] */
    1069 {
    1070
    1071   char cmd[MAXCMDLEN];
    1072
    1073   sprintf (cmd,"UPDATE %s
    1074                 SET prev_serial = %d
    1075                 WHERE object_id = %d
    1076                 AND sequence_id = %d",
    1077            tablename, prev_ser, obj_id, seq_id);
    1078
    1079   return(execute_sql_command(cmd));
    1080
    1081 } /* update_prev_serial_of_object() */
    1082
    1083
    1084
    1085 int update_serial_of_object (int obj_id, int seq_id, int ser_id, const char *tablename)
         /* [<][>][^][v][top][bottom][index][help] */
    1086 {
    1087
    1088   char cmd[MAXCMDLEN];
    1089
    1090   sprintf (cmd,"UPDATE %s
    1091                 SET serial = %d
    1092                 WHERE object_id = %d
    1093                 AND sequence_id = %d",
    1094            tablename, ser_id, obj_id, seq_id);
    1095
    1096   return(execute_sql_command(cmd));
    1097
    1098 } /* update_serial_of_object() */
    1099
    1100
    1101
    1102 int delete_serial_entry (int ser_id)
         /* [<][>][^][v][top][bottom][index][help] */
    1103 {
    1104
    1105   char cmd[MAXCMDLEN];
    1106
    1107   sprintf (cmd, "DELETE FROM serials WHERE serial_id = %d", ser_id);
    1108
    1109   return (execute_sql_command(cmd));
    1110
    1111 } /* delete_serial_entry() */
    1112
    1113
    1114 int delete_failed_transaction_entry (int ser_id)
         /* [<][>][^][v][top][bottom][index][help] */
    1115 {
    1116
    1117   char cmd[MAXCMDLEN];
    1118
    1119   sprintf (cmd, "DELETE FROM failed_transaction WHERE serial_id = %d",ser_id);
    1120
    1121   return (execute_sql_command(cmd));
    1122
    1123 } /* delete_failed_transaction_entry() */
    1124
    1125
    1126
    1127 int delete_entry_from_object_table (int obj_id, int seq_id, const char* tablename)
         /* [<][>][^][v][top][bottom][index][help] */
    1128 {
    1129
    1130   char cmd[MAXCMDLEN];
    1131   int affected_rows;
    1132
    1133   if (debug) printf ("Deleting %s entry. Objid: %d, seqid: %d\n",tablename, obj_id, seq_id);
    1134
    1135   sprintf (cmd, "DELETE FROM %s WHERE object_id = %d AND sequence_id = %d",
    1136            tablename, obj_id, seq_id);
    1137
    1138   affected_rows = execute_sql_command(cmd);
    1139   if (debug) printf ("delete_entry_from_object_table (%d, %d, %s): affected rows %d\n",obj_id,seq_id,tablename,affected_rows);
    1140   return (affected_rows);
    1141
    1142 } /* delete_entry_from_object_table() */
    1143
    1144
    1145 int fetch_timestamp_from_last (int obj_id, int seq_id)
         /* [<][>][^][v][top][bottom][index][help] */
    1146 {
    1147
    1148   int timestamp = 0;
    1149   char query[MAXCMDLEN];
    1150   SQ_result_set_t *result;
    1151   SQ_row_t *row;
    1152
    1153   sprintf (query, "SELECT timestamp FROM last WHERE object_id = %d AND sequence_id = %d",
    1154            obj_id, seq_id);
    1155
    1156   execute_sql_query(query, &result);
    1157   if ( (row = SQ_row_next(result)) != NULL)
    1158     timestamp = atoi((const char *)row[0]);
    1159
    1160   SQ_free_result(result);
    1161
    1162   return(timestamp);
    1163
    1164 }
    1165
    1166
    1167 int optimize_sql_table(const char* tablename)
         /* [<][>][^][v][top][bottom][index][help] */
    1168 {
    1169
    1170   char cmd[MAXCMDLEN];
    1171
    1172   sprintf (cmd, "OPTIMIZE TABLE %s", tablename);
    1173
    1174   return (execute_sql_command(cmd));
    1175
    1176 } /* optimize_sql_table() */
    1177
    1178
    1179 int execute_sql_query (const char *query, SQ_result_set_t **result_ptr)
         /* [<][>][^][v][top][bottom][index][help] */
    1180 {
    1181
    1182   int state;
    1183
    1184   state = SQ_execute_query(connection, query, result_ptr);
    1185   if (state != 0)
    1186     {
    1187       fprintf (stderr, "Fatal:\n Offending query: %s\n Error: %s\n",query,SQ_error(connection));
    1188       die;
    1189     }
    1190
    1191   return(state);
    1192
    1193 }
    1194
    1195
    1196
    1197 int execute_sql_command (const char *cmd)
         /* [<][>][^][v][top][bottom][index][help] */
    1198 {
    1199
    1200   int state;
    1201
    1202   state = SQ_execute_query(connection, cmd, NULL);
    1203   if (state != 0)
    1204     {
    1205       fprintf (stderr, "Fatal:\n Offending command: %s\n Error: %s\n",cmd,SQ_error(connection));
    1206       die;
    1207     }
    1208
    1209   return(SQ_get_affected_rows(connection));
    1210
    1211 }
    1212
    1213
    1214
    1215
    1216 void usage(char *argv[])
         /* [<][>][^][v][top][bottom][index][help] */
    1217 {
    1218
    1219   printf ("Usage: \n\n");
    1220   printf ("  %s [-?] [-h host] [-P port] [-u user] [-p password] [-d database]\n", argv[0]);
    1221   printf ("     [-T date|-S seconds|-M minutes|-H hours|-D days|-W weeks] \n");
    1222
    1223   printf ("\nGeneral options:\n");
    1224   printf ("   -?: This text\n");
    1225
    1226   printf ("\nSQL Options:\n");
    1227   printf ("   -h: host \t\t(default: %s)\n",MYSQL_HOST);
    1228   printf ("   -P: port \t\t(default: %d)\n",MYSQL_PORT);
    1229   printf ("   -u: user \t\t(default: %s)\n",MYSQL_USER);
    1230   printf ("   -p: password \t(default: %s)\n",MYSQL_PSWD);
    1231   printf ("   -d: database name \t(default: %s)\n",MYSQL_DB);
    1232
    1233   printf ("\nTime-related options: (one and only one must be specified)\n");
    1234   printf ("   -T date: Date before which to archive (secs from the Epoch)\n");
    1235   printf ("   -S seconds: Seconds elapsed between the date to archive and now\n");
    1236   printf ("   -M minutes: Minutes elapsed between the date to archive and now\n");
    1237   printf ("   -H hours: Hours elapsed between the date to archive and now\n");
    1238   printf ("   -D days: Days elapsed between the date to archive and now\n");
    1239   printf ("   -W weeks: Weeks elapsed between the date to archive and now\n");
    1240   exit(1);
    1241
    1242 }
    1243
    1244

/* [<][>][^][v][top][bottom][index][help] */