1 | /***************************************
2 | $Revision: 1.53 $
3 |
4 | Functions to process data stream( file, network socket, etc.)
5 |
6 | Status: NOT REVUED, NOT TESTED
7 |
8 | Author(s): Chris Ottrey, Andrei Robachevsky
9 |
10 | ******************/ /******************
11 | Modification History:
12 | andrei (17/01/2000) Created.
13 | ******************/ /******************
14 | Copyright (c) 2000 RIPE NCC
15 |
16 | All Rights Reserved
17 |
18 | Permission to use, copy, modify, and distribute this software and its
19 | documentation for any purpose and without fee is hereby granted,
20 | provided that the above copyright notice appear in all copies and that
21 | both that copyright notice and this permission notice appear in
22 | supporting documentation, and that the name of the author not be
23 | used in advertising or publicity pertaining to distribution of the
24 | software without specific, written prior permission.
25 |
26 | THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
27 | ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS; IN NO EVENT SHALL
28 | AUTHOR BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
29 | DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
30 | AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
31 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
32 | ***************************************/
33 | #include <sys/types.h>
34 | #include <sys/socket.h>
35 | #include <netdb.h>
36 | #include <arpa/inet.h>
37 | #include <unistd.h>
38 | #include <sys/stat.h>
39 | #include <fcntl.h>
40 | #include <string.h>
41 | #include "constants.h"
42 | #include "query_command.h"
43 | #include "ud.h"
44 | #include "ud_int.h"
45 | #include "ud_tr.h"
46 | #include "timediff.h"
47 |
48 | typedef enum _Line_Type_t {
49 | LINE_ATTRIBUTE,
50 | LINE_COMMENT,
51 | LINE_EMPTY,
52 | LINE_EOF,
53 | LINE_ADD,
54 | LINE_UPD,
55 | LINE_DEL,
56 | LINE_OVERRIDE_ADD,
57 | LINE_OVERRIDE_UPD,
58 | LINE_OVERRIDE_DEL,
59 | LINE_ACK
60 | } Line_Type_t;
61 |
62 | /* Maximum number of objects(serials) we can consume at a time */
63 | #define SBUNCH 1000
64 |
65 | static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason);
66 | static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
67 | static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation);
68 | static int process_transaction(UD_stream_t *ud_stream, Obj_parse_t *parse, int operation, long transaction_id);
69 |
70 | static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr);
71 | static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse);
72 | static void ud_parse_free(Obj_parse_t *parse);
73 | static int line_continuation(char *line);
74 | static GString *escape_apostrophes(GString *text);
75 | static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff);
76 | static int ud_replace_substring(GString *gstring, char *old, char *new);
77 | static int ud_normalize(Obj_parse_t *parse);
78 |
79 | /* Delimiters that separate list members, both RPS(,) and legacy( ) */
80 | #define ATTR_DELIMITERS " ,\t"
81 |
82 |
83 | static void ud_parse_init(SQ_connection_t *sql_connection, Obj_parse_t *parse){
84 | bzero(parse, sizeof(Obj_parse_t));
85 | parse->garbage = 0;
86 | parse->sql_connection = sql_connection;
87 | }
88 |
89 | static void ud_parse_free(Obj_parse_t *parse){
90 | UT_free(parse->nic);
91 | UT_free(parse->object_name);
92 | }
93 |
94 |
95 | /******************************************************************
96 | * int line_continuation() *
97 | * *
98 | * Checks if the line starts with one of line continuation signs *
99 | * Returns 1 if this is line continuation, 0 otherwise *
100 | * *
101 | * ****************************************************************/
102 | static int line_continuation(char *line)
103 | {
104 | switch(*line) {
105 | case ' ':
106 | case '\t':
107 | case '+':
108 | return(1); /* these indicate line continuation */
109 | default: return(0);
110 | }
111 |
112 | }
113 |
114 | /******************************************************************
115 | * int ud_replace_substring() *
116 | * *
117 | * replace a subsring in a string *
118 | * Returns -1 if no match was found *
119 | * *
120 | * ****************************************************************/
121 | static int ud_replace_substring(GString *gstring, char *old, char *new)
122 | {
123 | int i, i0=-1, j=0;
124 |
125 | for (i=0; ((i < gstring->len) && (old[j] != '\0')); i++){
126 | if(gstring->str[i]==old[j]){
127 | if(i0==-1)i0=i;
128 | j++;
129 | } else {
130 | i0=-1;
131 | j=0;
132 | }
133 | }
134 | /* if we haven't reached the end of the old substring */
135 | /* or no match occured return -1 */
136 | if((old[j] != '\0') || (i0==-1)) return (-1);
137 | g_string_erase(gstring, i0, j);
138 | g_string_insert(gstring, i0, new);
139 | return(0);
140 | }
141 |
142 | /************************************************************
143 | * *
144 | * The function to splits attribute value into multiple *
145 | * words and appends them as attr_type - attr_valu pairs *
146 | * to the attr_list *
147 | * *
148 | * *
149 | ************************************************************/
150 | static GSList *ud_split_attribute(GSList *attr_list, Attribute_t *attr)
151 | {
152 | char *token;
153 | char *split;
154 | char *value, *n;
155 | Attribute_t *attr_split;
156 | GSList *the_list = attr_list;
157 |
158 |
159 | /* check for line continuation (+) */
160 | if (strncmp(attr->value, "+", 1) == 0) attr->value++;
161 | /* check for end-of-line comments */
162 | n = index(attr->value, '#');
163 | /* if there is no comment check for trailing \n */
164 | if(n == NULL) n = index(attr->value, '\n');
165 | /* now copy the clean value into the attribute */
166 | if(n == NULL) value = g_strdup(attr->value);
167 | else value = g_strndup(attr->value, (n - attr->value));
168 |
169 | token=value;
170 | while((split=strsep(&token, ATTR_DELIMITERS))){
171 | if (*split != '\0'){
172 | attr_split = attribute_new1(attr->type, split);
173 | if (attr_split) the_list = g_slist_append(the_list, attr_split);
174 | }
175 | }
176 | free(value);
177 | attribute_free(attr, NULL);
178 |
179 | return(the_list);
180 | }
181 |
182 | /* XXX */
183 | static void each_attribute_print(void *element_data, void *tr_ptr)
184 | {
185 |
186 | Attribute_t *attr = (Attribute_t *)element_data;
187 |
188 | fprintf(stderr, "[%d|%s]\n", attr->type, attr->value);
189 |
190 | }
191 |
192 | /* XXX */
193 | static void print_object(Object_t *obj)
194 | {
195 | g_slist_foreach(obj->attributes, each_attribute_print, NULL);
196 | fprintf(stderr, ">>>>>\n%s\n", obj->object->str);
197 | }
198 |
199 |
200 | /******************************************************************
201 | * GString *escape_apostrophes() *
202 | * Escapes apostrophes in the text so they do not confuse printf *
203 | * functions and don't corrupt SQL queries *
204 | * *
205 | * *****************************************************************/
206 | static GString *escape_apostrophes(GString *text) {
207 | int i;
208 | for (i=0; i < text->len; i++) {
209 | if ((text->str[i] == '\'') || (text->str[i] == '\\')) {
210 | text = g_string_insert_c(text, i, '\\');
211 | i++;
212 | }
213 | }
214 | return(text);
215 | } /* escape_apostrophes() */
216 |
217 |
218 | /******************************************************************
219 | * Line_Type_t line_type(e) *
220 | * Determines the line type analysing the first letters *
221 | * *
222 | * ****************************************************************/
223 | static Line_Type_t line_type(const char *line, long *transaction_id) {
224 |
225 | if (strncmp(line, "# EOF", 4) == 0) return(LINE_EOF);
226 | if (strncmp(line, "#", 1) == 0) return(LINE_COMMENT);
227 | if (strcmp(line, "\n") == 0) return(LINE_EMPTY);
228 |
229 | if (strncmp(line, "ACK", 3) == 0) {
230 | *transaction_id = atol(line+3);
231 | return(LINE_ACK);
232 | }
233 | if (strncmp(line, "ADD_OVERRIDE", 12) == 0) {
234 | *transaction_id = atol(line+12);
235 | return(LINE_OVERRIDE_ADD);
236 | }
237 | if (strncmp(line, "UPD_OVERRIDE", 12) == 0) {
238 | *transaction_id = atol(line+12);
239 | return(LINE_OVERRIDE_UPD);
240 | }
241 | if (strncmp(line, "DEL_OVERRIDE", 12) == 0) {
242 | *transaction_id = atol(line+12);
243 | return(LINE_OVERRIDE_DEL);
244 | }
245 |
246 | if (strncmp(line, "ADD", 3) == 0) {
247 | *transaction_id = atol(line+3);
248 | return(LINE_ADD);
249 | }
250 | if (strncmp(line, "UPD", 3) == 0) {
251 | *transaction_id = atol(line+3);
252 | return(LINE_UPD);
253 | }
254 | if (strncmp(line, "DEL", 3) == 0) {
255 | *transaction_id = atol(line+3);
256 | return(LINE_DEL);
257 | }
258 |
259 | /* Otherwise this is an attribute */
260 | return(LINE_ATTRIBUTE);
261 |
262 | } /* line_type() */
263 |
264 | /******************************************************************
265 | * Object_t *UD_parse_object() *
266 | * *
267 | * Parses the object accepting line by line *
268 | * Stores the object and list of attributes in Obj_parse_t struct *
269 | * *
270 | * ****************************************************************/
271 | static Object_t *UD_parse_object(Obj_parse_t *parse, char *line_buff)
272 | {
273 | GString *g_line_buff;
274 | Attribute_t *attr;
275 |
276 | /* in case of garbage */
277 | if(parse->garbage) return(NULL);
278 |
279 | if(parse->obj==NULL) {
280 | /* which means this is the start - create a new object*/
281 | parse->obj = object_new(line_buff);
282 | /* if this is not a class attribute - this is a garbage till the next blank line */
283 | if(parse->obj==NULL) {
284 | parse->garbage = 1;
285 | return(NULL);
286 | }
287 | }
288 |
289 | /* allocate buffer or the string */
290 | if ((g_line_buff = g_string_sized_new(STR_XXL)) == NULL){
291 | ER_perror(FAC_UD, UD_MEM, "cannot allocate gstring");
292 | die;
293 | }
294 | /* copy the input */
295 | g_string_sprintf(g_line_buff, "%s", line_buff);
296 | /* escape apostrophes in the input line */
297 | g_line_buff=escape_apostrophes(g_line_buff);
298 |
299 | attr = attribute_new(g_line_buff->str);
300 | if (attr){
301 | /* save this attribute for future line continuations */
302 | /* skip attributes that are not stored in SQL database */
303 | if(DF_get_update_query_type(attr->type)==UD_NULL_){
304 | parse->current_attr = NULL;
305 | attribute_free(attr, NULL);
306 | }
307 | else {
308 | parse->current_attr = attr;
309 | if((attr->type == A_MB) || (attr->type == A_NH)) {
310 | /* insert it at the beginning */
311 | /* mnt-by should go before member-of to allow correct membership autorization (still done in RIPupd) */
312 | /* nic-hdl should go before any admin-c, tech-c to prevent errrors in self referencing role objects */
313 | parse->obj->attributes = g_slist_insert(parse->obj->attributes, parse->current_attr, 1);
314 | } else {
315 | /* append it to the attribute list */
316 | parse->obj->attributes = g_slist_append(parse->obj->attributes, parse->current_attr);
317 | }
318 | }
319 | }
320 | /* if this is not a valid attribute, then this may be unknown attr or line continuation */
321 | else
322 | if(line_continuation(g_line_buff->str))
323 | {
324 | /* if parse->attr == NULL, then this attr is not stored in SQL database - skip it */
325 | if(parse->current_attr){
326 | /* this is continuation of a previous attribute */
327 | char *line_value, *new_value;
328 | char *n;
329 | /* normalize and append to current_attr->value */
330 | /* check for end-of-line comments */
331 | n = index(g_line_buff->str, '#');
332 | /* if there is no comment check for trailing \n */
333 | if(n == NULL) n = index(g_line_buff->str, '\n');
334 | /* now copy the clean value into the attribute */
335 | if(n == NULL) line_value = g_strdup(g_line_buff->str);
336 | else line_value = g_strndup(g_line_buff->str, (n - g_line_buff->str));
337 | /* replace line continuation character with ' ' */
338 | *line_value = ' ';
339 | new_value = g_strconcat(parse->current_attr->value, line_value, NULL);
340 | parse->current_attr = attribute_upd(parse->current_attr, parse->current_attr->type, new_value);
341 | UT_free(new_value);
342 | UT_free(line_value);
343 | }
344 | }
345 | /* otherwise, this is unknown attribute, so we need to break the continuation */
346 | else parse->current_attr=NULL;
347 | /* copy the line into object no matter whether it is an attribute or not (continualtion, etc.) */
348 | g_string_sprintfa(parse->obj->object, "%s", g_line_buff->str);
349 | g_string_free(g_line_buff, TRUE);
350 | return(parse->obj);
351 | }
352 |
353 | /******************************************************************
354 | * int each_attribute_2_pass() *
355 | * *
356 | * Assigns nic-handles in case of AUTO *
357 | * Splits attributes *
358 | * * *
359 | ******************************************************************/
360 | void each_attribute_2_pass(void *element_data, void *ptr)
361 | {
362 | Attribute_t *attr = (Attribute_t *)element_data;
363 | Obj_parse_t *parse = (Obj_parse_t *)ptr;
364 |
365 |
366 | /* Strip the white space */
367 | g_strstrip(attr->value);
368 |
369 | switch(attr->type){
370 | case A_NH: /* nic-hdl */
371 | /* Parse the string into nh structure */
372 | /* In case of an AUTO NIC handle check the ID in the database */
373 | /* Possible errors leave to core processing */
374 | if(NH_parse(attr->value, &parse->nh_ptr) == 0) {
375 | /* this is an AUTO nic handle */
376 | /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsing nic handle: [%s]", UD_TAG, attr->value);*/
377 | /* Check if we can allocate it */
378 | if(NH_check(parse->nh_ptr, parse->sql_connection)>0){
379 | /* Convert nh to the database format */
380 | parse->nic = NH_convert(parse->nh_ptr);
381 | /* ER_dbg_va(FAC_UD, ASP_UD_OBJ, "%s parsed and converted nic handle: [%s]", UD_TAG, nic); */
382 | /* replace the auto nic with a real one */
383 | if(ud_replace_substring(parse->obj->object, attr->value, parse->nic)==-1) {
384 | ER_perror(FAC_UD, UD_BUG, "cannot replace auto nic handle");
385 | die;
386 | }
387 | /* Update the attribute */
388 | attr = attribute_upd(attr, attr->type, parse->nic);
389 |
390 | }
391 | }
392 | parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
393 | break;
394 | case A_PN: /* person */
395 | case A_RO: /* role */
396 | case A_MR: /* mbrs-by-ref */
397 | case A_MB: /* mnt-by */
398 | case A_ML: /* mnt-lower */
399 | case A_CT: /* cross-mnt */
400 | case A_MO: /* member-of */
401 | case A_SD: /* sub-dom */
402 | case A_RZ: /* rev-srv */
403 | case A_NS: /* nserver */
404 | parse->new_attr_list = ud_split_attribute(parse->new_attr_list, attr);
405 | break;
406 | default:
407 | parse->new_attr_list = g_slist_append(parse->new_attr_list, attr);
408 | break;
409 | }
410 |
411 | }
412 |
413 |
414 | /******************************************************************
415 | * int ud_normalize() *
416 | * *
417 | * function accepts Obj_parse_t structure with attribute list *
418 | * *
419 | * It makes another pass to normalize object *
420 | * Normalization includes: *
421 | * *
422 | * 1) reorder attributes *
423 | * . mnt-by should go before member-of to allow correct *
424 | * membership autorization (still done in RIPupd) *
425 | * . nic-hdl should go before any admin-c, tech-c to prevent *
426 | * errrors in self referencing role objects *
427 | * 2) in case of pn/ro check nic handle and replace it in *
428 | * the obj->object if needed (AUTO) *
429 | * 3) split some attributes that allow lists *
430 | ******************************************************************/
431 | static int ud_normalize(Obj_parse_t *parse)
432 | {
433 | GSList *old_attr_list;
434 | GSList *first_element;
435 | Attribute_t *class_attr;
436 |
437 | old_attr_list = parse->obj->attributes;
438 | /* get class attribute - the first one */
439 | first_element = g_slist_nth(old_attr_list, 0);
440 | class_attr = (Attribute_t *)first_element->data;
441 | /* save object name for reporting and checking ref integrity for names (legacy) */
442 | parse->object_name = g_strdup(class_attr->value);
443 | /* reorder the attributes mnt-by and nic-hdl come first */
444 | /*g_slist_foreach(old_attr_list, each_attribute_1_pass, parse);
445 | g_slist_free(old_attr_list);
446 | old_attr_list = parse->new_attr_list; */
447 | parse->new_attr_list = NULL;
448 | /* assign auto nic-hdl and slit attributes */
449 | g_slist_foreach(old_attr_list, each_attribute_2_pass, parse);
450 | g_slist_free(old_attr_list);
451 | parse->obj->attributes = parse->new_attr_list;
452 |
453 | return(1);
454 | }
455 |
456 |
457 | /******************************************************************
458 | * report_transaction() *
459 | * *
460 | * Prints error report to the log *
461 | * *
462 | * reason - additional message that will be included *
463 | * *
464 | * *****************************************************************/
465 | static int report_transaction(Transaction_t *tr, long transaction_id, Log_t *log, ut_timer_t *psotime, char *reason)
466 | {
467 | int result=0;
468 | ut_timer_t fotime;
469 | float timediff;
470 | const char *class_name = DF_class_type2name(tr->class_type);
471 | char *primary_key = tr->K->str;
472 |
473 |
474 | /* calculate statistics */
475 | UT_timeget(&fotime);
476 | timediff = UT_timediff(psotime, &fotime);
477 |
478 | if(tr->succeeded==0) {
479 | result=tr->error;
480 | log->num_failed++;
481 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] %.2fs FAILED [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
482 | /* ER_inf_va(FAC_UD, ASP_UD_UPDLOG, "[%ld] object: FAILED [%s][%s](%d/%d)", transaction_id, , reason, log->num_failed, (log->num_failed)+(log->num_ok)); */
483 | if(result & ERROR_U_OBJ) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: referential integrity error", transaction_id);
484 | if(result & ERROR_U_AUT) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: authentication error", transaction_id);
485 | if(result & ERROR_U_BADOP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: unsupported operation", transaction_id);
486 | if(result & ERROR_U_COP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: conflicting operation", transaction_id);
487 | if(result & ERROR_U_NSUP) ER_dbg_va(FAC_UD, ASP_UD_OBJ,"[%ld] object: this type is not supported", transaction_id);
488 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
489 | result=1; /* # of failures */
490 | }
491 | else {
492 | result=0;
493 | log->num_ok++;
494 | /* ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: OK [%s](%d/%d)", transaction_id, obj_name, log->num_ok, (log->num_failed)+(log->num_ok)); */
495 | ER_inf_va(FAC_UD, ASP_UD_OBJ, "[%ld] %.2fs OK [%s:%s][%s]", transaction_id, timediff, class_name, primary_key, reason);
496 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: details [%s]", transaction_id, (tr->error_script)->str);
497 | }
498 |
499 | return(result);
500 | }/* report_transaction() */
501 |
502 |
503 |
504 | /************************************************************
505 | * process_nrtm() *
506 | * *
507 | * Process object in NRTM client mode *
508 | * *
509 | * nrtm - pointer to _nrtm structure *
510 | * log - pointer to Log_t structure *
511 | * object_name - name of the object *
512 | * operation - operation code (OP_ADD/OP_DEL) *
513 | * *
514 | * Returns: *
515 | * 1 - okay *
516 | * <0 - error *
517 | * *
518 | ************************************************************/
519 |
520 | static int process_nrtm(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
521 | {
522 | int result=0;
523 | int dummy=0;
524 | struct _nrtm *nrtm = ud_stream->nrtm;
525 | long serial_id;
526 | Log_t *log_ptr= &(ud_stream->log);
527 | ut_timer_t sotime;
528 | int ta_upd_nhr;
529 |
530 | /* Start timer for statistics */
531 | UT_timeget(&sotime);
532 |
533 | /* We allow NRTM updates for some inconsistent objects */
534 | /* One of the examples is reference by name which looks like nic-handle */
535 | /* For this purpose we allow dummy creation when updating an object */
536 | /* We also check for dummy allowance when deleting an object */
537 | /* this is done to allow deletion of person objects referenced by name */
538 |
539 | tr->mode|=B_DUMMY;
540 | if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
541 |
542 | switch (operation) {
543 |
544 | case OP_ADD:
545 | if(nrtm->tr){ /* DEL ADD => saved*/
546 | if(tr->object_id==0) {
547 | /* object does not exist in the DB */
548 | /* delete the previous(saved) object*/
549 | object_process(nrtm->tr);
550 | /* create DEL serial */
551 | UD_lock_serial(nrtm->tr);
552 | serial_id = UD_create_serial(nrtm->tr);
553 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
554 | UD_commit_serial(nrtm->tr);
555 | UD_unlock_serial(nrtm->tr);
556 | /* Mark TR as clean */
557 | TR_mark_clean(nrtm->tr);
558 | /* log the transaction */
559 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
560 |
561 | object_free(nrtm->tr->object);
562 | transaction_free(nrtm->tr); nrtm->tr=NULL;
563 |
564 | /* Create an object and update NHR */
565 | tr->action=(TA_CREATE | ta_upd_nhr);
566 | /* restart the timer for statistics */
567 | UT_timeget(&sotime);
568 | object_process(tr); /* create a new one*/
569 | /* create ADD serial */
570 | UD_lock_serial(tr);
571 | serial_id = UD_create_serial(tr);
572 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
573 | UD_commit_serial(tr);
574 | UD_unlock_serial(tr);
575 | /* Mark TR as clean */
576 | TR_mark_clean(tr);
577 | /* log the transaction */
578 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
579 | }
580 | else {
581 | /* object already exists in the DB - update or dummy replacement*/
582 | /*compare the two, may be we may collapse operations*/
583 | if(tr->object_id==nrtm->tr->object_id) {
584 | /* DEL-ADD ->> UPDATE */
585 | object_free(nrtm->tr->object);
586 | transaction_free(nrtm->tr); nrtm->tr=NULL;
587 | tr->action=TA_UPD_CLLPS;
588 | object_process(tr);
589 | /* create DEL+ADD serial records */
590 | UD_lock_serial(tr);
591 | tr->action=TA_DELETE; serial_id = UD_create_serial(tr);
592 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL(UPD)");
593 |
594 | /* restart the timer for statistics */
595 | UT_timeget(&sotime);
596 | tr->sequence_id++;
597 | tr->action=TA_CREATE; serial_id = UD_create_serial(tr);
598 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
599 | UD_commit_serial(tr);
600 | UD_unlock_serial(tr);
601 | /* Mark TR as clean */
602 | TR_mark_clean(tr);
603 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD(UPD)");
604 | }
605 | else { /* this should be a dummy object in the database(that we are going to replace with the real one */
606 | /* or an interleaved operation*/
607 | object_process(nrtm->tr); /* delete the previous(saved) object*/
608 | /* create a DEL serial record */
609 | UD_lock_serial(nrtm->tr);
610 | serial_id = UD_create_serial(nrtm->tr);
611 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
612 | UD_commit_serial(nrtm->tr);
613 | UD_unlock_serial(nrtm->tr);
614 | /* Mark TR as clean */
615 | TR_mark_clean(nrtm->tr);
616 | /* result=report_transaction(nrtm->tr, log_ptr, nrtm->object_name, "NRTM:DEL:While deleting previous(saved)");*/
617 | /* log the transaction */
618 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
619 |
620 |
621 | object_free(nrtm->tr->object);
622 | transaction_free(nrtm->tr); nrtm->tr=NULL;
623 |
624 | /* restart the timer for statistics */
625 | UT_timeget(&sotime);
626 |
627 | tr->action=TA_UPDATE;
628 | /* check if we are replacing a dummy object */
629 | dummy=isdummy(tr);
630 | /* If we are replacing dummy with a real object update NHR */
631 | if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
632 | /* fprintf(stderr,"UPDATE next(dummy)\n"); */
633 | object_process(tr); /* create a new one*/
634 | /* result=report_transaction(tr, log_ptr, object_name, "NRTM:ADD:While creating new"); */
635 | /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
636 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
637 | /* create ADD serial record */
638 | UD_lock_serial(tr);
639 | serial_id = UD_create_serial(tr);
640 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
641 | UD_commit_serial(tr);
642 | UD_unlock_serial(tr);
643 | /* Mark TR as clean */
644 | TR_mark_clean(tr);
645 | /* log the transaction */
646 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
647 |
648 | }
649 | }
650 | }
651 | else { /* ADD ADD =>brand new object*/
652 | if(tr->object_id==0) {
653 | /* fprintf(stderr,"CREATE new\n");*/
654 | /* Create an object and update NHR */
655 | tr->action=(TA_CREATE | ta_upd_nhr);
656 | object_process(tr);
657 | /* create ADD serial */
658 | UD_lock_serial(tr);
659 | serial_id = UD_create_serial(tr);
660 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
661 | UD_commit_serial(tr);
662 | UD_unlock_serial(tr);
663 |
664 | /* Mark TR as clean */
665 | TR_mark_clean(tr);
666 | /* log the transaction */
667 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
668 | }
669 | else { /* object already exists in the database */
670 | /* this may happen because of dummies*/
671 | /* or with some implementations of mirroring protocol that have atomic update */
672 | /* instead of add + del */
673 | tr->action=TA_UPDATE;
674 | dummy=isdummy(tr);
675 | /* If we are replacing dummy with a real object update NHR */
676 | if(dummy==1) tr->action = ( ta_upd_nhr| TA_UPD_DUMMY);
677 | object_process(tr);
678 | /* For serials this is CREATE operation. Increase sequence to have it correct in serals */
679 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
680 | /* create ADD serial record */
681 | UD_lock_serial(tr);
682 | serial_id = UD_create_serial(tr);
683 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
684 | UD_commit_serial(tr);
685 | UD_unlock_serial(tr);
686 | /* Mark TR as clean */
687 | TR_mark_clean(tr);
688 | /* log the transaction */
689 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:ADD");
690 |
691 | }
692 | }
693 | break;
694 |
695 | case OP_DEL:
696 | if(nrtm->tr){ /*DEL DEL =>saved */
697 | /* check this is not a deletion of the same object twise */
698 | /* this should not happen but we cannot trust foreign sources */
699 | /* in such case process saved transaction but fail the current one */
700 | if(nrtm->tr->object_id == tr->object_id) tr->object_id=0;
701 |
702 | object_process(nrtm->tr); /* delete the previous(saved) object*/
703 | /* create DEL serial record */
704 | UD_lock_serial(nrtm->tr);
705 | serial_id = UD_create_serial(nrtm->tr);
706 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
707 | UD_commit_serial(nrtm->tr);
708 | UD_unlock_serial(nrtm->tr);
709 | /* Mark TR as clean */
710 | TR_mark_clean(nrtm->tr);
711 | /* log the transaction */
712 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
713 |
714 | object_free(nrtm->tr->object);
715 | transaction_free(nrtm->tr); nrtm->tr=NULL;
716 | }
717 | /* save the real object (not a dummy one ) */
718 | if(tr->object_id>0 && !isdummy(tr)){
719 | /* save the object*/
720 | tr->action=(TA_DELETE | ta_upd_nhr);
721 | nrtm->tr=tr;
722 | return(0);
723 | }
724 | else { /* this is an error - Trying to DEL non-existing object*/
725 | tr->succeeded=0; tr->error|=ERROR_U_COP;
726 | tr->action=(TA_DELETE | ta_upd_nhr);
727 | /* create and initialize TR record for crash recovery */
728 | TR_create_record(tr);
729 | /* create DEL serial record anyway */
730 | UD_lock_serial(tr);
731 | serial_id = UD_create_serial(tr);
732 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
733 | UD_commit_serial(tr);
734 | UD_unlock_serial(tr);
735 | /* Mark TR as clean */
736 | TR_mark_clean(tr);
737 | /* log the transaction */
738 | result=report_transaction(tr, serial_id, log_ptr, &sotime, "M:DEL: non-existing object");
739 |
740 | }
741 | break;
742 |
743 | default:
744 | tr->succeeded=0; tr->error |=ERROR_U_BADOP;
745 | break;
746 | }
747 |
748 | /* Free resources */
749 | object_free(tr->object);
750 | transaction_free(tr);
751 |
752 | return(result);
753 | } /* process_nrtm() */
754 |
755 |
756 |
757 | /************************************************************
758 | * process_updates() *
759 | * *
760 | * Process object in update mode *
761 | * *
762 | * ud_stream - pointer to UD_stream structure *
763 | * object_name - name of the object *
764 | * operation - operation code (OP_ADD/OP_DEL) *
765 | * *
766 | * Note: *
767 | * Frees tr and tr->obj on exit *
768 | * *
769 | * Returns: *
770 | * 0 - okay *
771 | * <0- number of failed objects *
772 | * *
773 | ************************************************************/
774 |
775 | static int process_updates(UD_stream_t *ud_stream, Transaction_t *tr, int operation)
776 | {
777 | int result=0;
778 | Log_t *log_ptr= &(ud_stream->log);
779 | int dummy=0;
780 | ut_timer_t sotime;
781 | int ta_upd_nhr;
782 | char *reason;
783 |
784 | /* Start timer for statistics */
785 | UT_timeget(&sotime);
786 | if(IS_NO_NHR(tr->mode))ta_upd_nhr=0; else ta_upd_nhr = TA_UPD_NHR;
787 |
788 | switch(operation) {
789 | /* Compare operations and report an error if they do not match */
790 | case OP_ADD:
791 | if(tr->object_id!=0) { /* trying to create, but object exists */
792 | tr->succeeded=0; tr->error|=ERROR_U_COP;
793 | reason="U:ADD:object already exists";
794 | UD_ack(tr); /* Send a NACK */
795 | } else {
796 | /* Action: create the object and update NHR */
797 | tr->action=(TA_CREATE | ta_upd_nhr);
798 | reason="U:ADD";
799 | object_process(tr);
800 | }
801 | break;
802 | case OP_UPD:
803 | if(tr->object_id==0) { /* trying to update non-existing object*/
804 | tr->succeeded=0; tr->error|=ERROR_U_COP;
805 | reason="U:UPD:non-existing object";
806 | UD_ack(tr); /* Send a NACK */
807 | } else {
808 | tr->action=TA_UPDATE;
809 | reason="U:UPD";
810 | dummy=isdummy(tr);
811 | /* If we are replacing dummy with a real object update NHR */
812 | if(dummy==1) tr->action = (ta_upd_nhr | TA_UPD_DUMMY);
813 | object_process(tr);
814 | }
815 | break;
816 |
817 | case OP_DEL:
818 | if(tr->object_id==0) { /* trying t delete non-existing object*/
819 | tr->succeeded=0; tr->error|=ERROR_U_COP;
820 | reason="U:DEL:non-existing object";
821 | UD_ack(tr);
822 | } else {
823 | tr->action=(TA_DELETE | ta_upd_nhr);
824 | reason="U:DEL";
825 | object_process(tr);
826 | }
827 | break;
828 |
829 | default:
830 | /* bad operation for this mode if not standalone */
831 | if(IS_STANDALONE(tr->mode)) {
832 | if(tr->object_id==0){
833 | tr->action=(TA_CREATE | ta_upd_nhr);
834 | reason="U:ADD";
835 | }
836 | else {
837 | tr->action=TA_UPDATE;
838 | reason="U:UPD";
839 | }
840 | object_process(tr);
841 | }
842 | else {
843 | tr->succeeded=0;
844 | tr->error|=ERROR_U_BADOP;
845 | reason="U:bad operation";
846 | UD_ack(tr); /* Send a NACK */
847 | }
848 | break;
849 | }
850 | /* If not in standalone mode create serial and copy error transcript */
851 | if(!IS_STANDALONE(tr->mode)) {
852 | if(tr->succeeded){
853 | /* we don't want to generate DEL serial for dummy replacement*/
854 | if(dummy==1) { tr->action=TA_CREATE; tr->sequence_id++; }
855 | UD_lock_serial(tr);
856 | UD_create_serial(tr);
857 | CP_CREATE_S_PASSED(tr->action); TR_update_status(tr);
858 | UD_commit_serial(tr);
859 | UD_unlock_serial(tr);
860 | /* Mark the TR as clean */
861 | TR_mark_clean(tr);
862 | }
863 | }
864 |
865 | /* Make a report. U stands for update stream. No reason */
866 | result=report_transaction(tr, tr->transaction_id, log_ptr, &sotime, reason);
867 |
868 | /* Free resources */
869 | object_free(tr->object);
870 | transaction_free(tr);
871 |
872 | return(result);
873 |
874 | } /* process_updates() */
875 |
876 |
877 | /************************************************************
878 | * *
879 | * int process_transaction() *
880 | * *
881 | * Processes the transaction *
882 | * *
883 | * ud_stream - pointer to UD_stream_t structure *
884 | * *
885 | * Returns: *
886 | * 0 - no error *
887 | * <0- number of failed objects *
888 | * *
889 | ************************************************************/
890 |
891 | /* It frees the obj */
892 |
893 | static int process_transaction(UD_stream_t *ud_stream,
894 | Obj_parse_t *parse,
895 | int operation,
896 | long transaction_id)
897 | {
898 | Transaction_t *tr = NULL;
899 | Object_t *obj = parse->obj;
900 | int result;
901 |
902 | /* check if the requested transaction has already been processed */
903 | /* this may happen in case of crash. If so, just send an ack and return */
904 | if(TR_check(ud_stream->db_connection, transaction_id, (ud_stream->condat).sock))return(1);
905 |
906 | /* start new transaction now */
907 | tr = transaction_new(ud_stream->db_connection, obj->type);
908 |
909 | /* Return with error if transaction cannot be created */
910 | if (tr == NULL) die;
911 |
912 | tr->mode=ud_stream->ud_mode;
913 | tr->load_pass=ud_stream->load_pass;
914 | tr->object=obj;
915 | tr->nh=parse->nh_ptr;
916 | tr->source_hdl=ud_stream->source_hdl;
917 | tr->socket=(ud_stream->condat).sock;
918 | tr->transaction_id=transaction_id;
919 |
920 | /* We perform no commit/rollback in the loader mode, so thread_id should be set to 0 */
921 | if(ud_stream->load_pass!=0) { tr->thread_ins=0; tr->thread_upd=0; }
922 |
923 | /* For the first load pass we only create objects */
924 | if(ud_stream->load_pass==1) {
925 | Object_t *obj=tr->object;
926 | /* still we need to fill tr->K (last.pkey) field */
927 | g_slist_foreach(obj->attributes, ud_each_primary_key_select, tr);
928 | tr->object_id=0;
929 | }
930 | else tr->object_id=get_object_id(tr);
931 |
932 | /* Object cannot be retrieved */
933 | if(tr->object_id==-1) { /* DB error*/
934 | tr->succeeded=0;
935 | tr->error |= ERROR_U_DBS;
936 | ER_perror(FAC_UD, UD_SQL, "%s: Object cannot be retrieved", parse->object_name);
937 | die;
938 | transaction_free(tr);
939 | object_free(obj);
940 | }
941 | /* save the name of person/role as we need it for referential */
942 | /* integrity check when deleting the object against names. */
943 | /* This is needed to support legacy references by name rather */
944 | /* then by nic_hdl */
945 | if((tr->class_type==C_PN) || (tr->class_type==C_RO)){
946 | /* Save the value */
947 | tr->save=g_strdup(parse->object_name);
948 | /* attribute_free(attr, NULL); */
949 | }
950 |
951 | /* Process transaction. tr and obj are freed inside the process_* functions */
952 |
953 | if(IS_UPDATE(ud_stream->ud_mode))
954 | /* We are in update mode */
955 | result=process_updates(ud_stream, tr, operation);
956 | else
957 | /* We are in NRTM mode */
958 | result=process_nrtm(ud_stream, tr, operation);
959 |
960 | return(result);
961 |
962 | }
963 |
964 |
965 | /************************************************************
966 | * *
967 | * int UD_process_stream(UD_stream_t *ud_stream) *
968 | * *
969 | * Processes the stream *
970 | * *
971 | * ud_stream - pointer to UD_stream_t structure *
972 | * *
973 | * Returns: *
974 | * in update mode (!standalone)(1 object processed): *
975 | * 1 - no error *
976 | * <0- errors *
977 | * *
978 | * in NRTM & standalone modes *
979 | * total number of object processed *
980 | * *
981 | ************************************************************/
982 |
983 | int UD_process_stream(UD_stream_t *ud_stream)
984 | {
985 | char line_buff[STR_XXL];
986 | Object_t *obj = NULL;
987 | SQ_connection_t *sql_connection;
988 | int start_object;
989 | int a_type;
990 | struct _nrtm *nrtm;
991 | Log_t *log_ptr= &(ud_stream->log);
992 | ut_timer_t stime, ftime, sotime;
993 | float obj_second1, obj_second10, timediff;
994 | int result;
995 | int operation=0;
996 | int interrupt=0;
997 | int do_update;
998 | int default_ud_mode = ud_stream->ud_mode;
999 | Line_Type_t linetype;
1000 | Transaction_t *tr;
1001 | long transaction_id=0; /* transaction_id (to be supplied by DBupdate and stored in Database) */
1002 | long serial_id;
1003 | Obj_parse_t obj_parse; /* the structure used to parse a text object */
1004 |
1005 |
1006 |
1007 |
1008 | nrtm=ud_stream->nrtm;
1009 | start_object = 1;
1010 | a_type=-1;
1011 |
1012 |
1013 | /* Check connection to the database */
1014 | if(SQ_ping(ud_stream->db_connection)) {
1015 | ER_perror(FAC_UD, UD_SQL, "%s", SQ_error(ud_stream->db_connection));
1016 | die;
1017 | }
1018 |
1019 | sql_connection=ud_stream->db_connection;
1020 |
1021 | ud_parse_init(sql_connection, &obj_parse);
1022 | /* Start timer for statistics */
1023 | UT_timeget(&stime);
1024 |
1025 | /* Main loop. Reading input stream line by line */
1026 | /* Empty line signals to start processing an object, if we have it */
1027 | while (SK_cd_gets(&ud_stream->condat, line_buff, sizeof(line_buff))>0) {
1028 |
1029 |
1030 | switch (linetype=line_type(line_buff, &transaction_id)) {
1031 | case LINE_ATTRIBUTE:
1032 | /* parse the object line by line */
1033 | obj = UD_parse_object(&obj_parse, line_buff);
1034 |
1035 | break;
1036 |
1037 | case LINE_COMMENT:
1038 | break;
1039 |
1040 | case LINE_EOF:
1041 | break;
1042 |
1043 | case LINE_ACK:
1044 | tr = transaction_new(ud_stream->db_connection, 0);
1045 | tr->transaction_id=transaction_id;
1046 | TR_delete_record(tr);
1047 | transaction_free(tr);
1048 | break;
1049 |
1050 |
1051 | case LINE_ADD:
1052 | /* restore the default operation mode */
1053 | operation=OP_ADD;
1054 | ud_stream->ud_mode=default_ud_mode;
1055 | break;
1056 |
1057 | case LINE_OVERRIDE_ADD:
1058 | /* for override - switch the dummy bit on */
1059 | operation=OP_ADD;
1060 | ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1061 | break;
1062 |
1063 | case LINE_UPD:
1064 | /* restore the default operation mode */
1065 | operation=OP_UPD;
1066 | ud_stream->ud_mode=default_ud_mode;
1067 | break;
1068 |
1069 | case LINE_OVERRIDE_UPD:
1070 | /* for override - switch the dummy bit on */
1071 | operation=OP_UPD;
1072 | ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1073 | break;
1074 |
1075 | case LINE_DEL:
1076 | /* restore the default operation mode */
1077 | operation=OP_DEL;
1078 | ud_stream->ud_mode=default_ud_mode;
1079 | break;
1080 |
1081 | case LINE_OVERRIDE_DEL:
1082 | /* for override - switch the dummy bit on */
1083 | operation=OP_DEL;
1084 | ud_stream->ud_mode=default_ud_mode|B_DUMMY;
1085 | break;
1086 |
1087 | case LINE_EMPTY:
1088 | /* start processing the object */
1089 | if ((obj=obj_parse.obj)) { /* if not just garbage*/
1090 | /* normalize the object (reorder and split attributes */
1091 | ud_normalize(&obj_parse);
1092 | /* XXX */
1093 | /* print_object(obj); */
1094 |
1095 | /* start new transaction now */
1096 | ER_dbg_va(FAC_UD, ASP_UD_OBJ, "[%ld] object: [%s] ", transaction_id, obj_parse.object_name);
1097 |
1098 | /* fprintf(stderr, "transction # %ld\n", transaction_id); */
1099 | result=process_transaction(ud_stream, &obj_parse, operation, transaction_id);
1100 | /* process_transaction() frees tr and obj structures, */
1101 | /* so make sure we'll not reference these objects in the future */
1102 | operation=OP_NOOP;
1103 | transaction_id=0;
1104 | ud_stream->ud_mode=default_ud_mode;
1105 | ud_parse_free(&obj_parse);
1106 |
1107 | /* this is a good place for quick interrupt */
1108 | do_update=CO_get_do_update();
1109 | if (do_update) interrupt=0; else interrupt=1;
1110 | } /* if this is a real object */
1111 | /* initialize the parsing structure */
1112 | ud_parse_init(sql_connection, &obj_parse);
1113 |
1114 | break;
1115 |
1116 | default:
1117 | die;
1118 | } /* switch */
1119 |
1120 | /* Finish processing if interrupt has been set */
1121 | if (interrupt) break;
1122 | } /* Main loop of data stream processing : while */
1123 |
1124 | /* Some postprocessing */
1125 | if(IS_NRTM_CLNT(ud_stream->ud_mode)){
1126 | /* We are in NRTM mode */
1127 | /* Clean up */
1128 | /* fclose(ud_stream->stream); */
1129 | /* In NRTM mode there may be a saved object that is unprocessed */
1130 | if(nrtm->tr){ /*saved backlog?*/
1131 | /* restart the timer for statistics */
1132 | UT_timeget(&sotime);
1133 | object_process(nrtm->tr); /* delete the previous(saved) object*/
1134 | /* result=report_transaction(nrtm->tr, &(ud_stream->log), nrtm->object_name,
1135 | "NRTM:DEL:While deleting previous(saved) object"); */
1136 | /* create DEL serial record no matter what the result is */
1137 | UD_lock_serial(nrtm->tr);
1138 | serial_id = UD_create_serial(nrtm->tr);
1139 | CP_CREATE_S_PASSED(nrtm->tr->action); TR_update_status(nrtm->tr);
1140 | UD_commit_serial(nrtm->tr);
1141 | UD_unlock_serial(nrtm->tr);
1142 | /* Mark TR as clean */
1143 | TR_mark_clean(nrtm->tr);
1144 | /* log the transaction */
1145 | result=report_transaction(nrtm->tr, serial_id, log_ptr, &sotime, "M:DEL");
1146 |
1147 | object_free(nrtm->tr->object);
1148 | transaction_free(nrtm->tr); nrtm->tr=NULL;
1149 | }
1150 | }
1151 |
1152 | /* That's all. Free GString */
1153 | /* g_string_free(g_line_buff, TRUE);*/
1154 |
1155 |
1156 | /* Calculate some statistics */
1157 | /* ftime=time(NULL); */
1158 | UT_timeget(&ftime);
1159 | /* obj_second1 = (float)(log_ptr->num_ok)/(ftime-stime);
1160 | obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/(ftime-stime); */
1161 | timediff = UT_timediff(&stime, &ftime);
1162 | obj_second1 = (float)(log_ptr->num_ok)/timediff;
1163 | obj_second10 = (float)(log_ptr->num_ok+log_ptr->num_failed)/timediff;
1164 |
1165 | /* Print the report */
1166 | if(IS_STANDALONE(ud_stream->ud_mode) || (!IS_UPDATE(ud_stream->ud_mode))) {
1167 |
1168 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s ******** report **********", UD_TAG);
1169 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects OK (%7.4f obj/s)", UD_TAG, log_ptr->num_ok, obj_second1);
1170 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s %d objects failed", UD_TAG, log_ptr->num_failed);
1171 | ER_inf_va(FAC_UD, ASP_UD_UPDLOG,"%s average processing time %7.4f obj/s (%6.2f obj/min)", UD_TAG,
1172 | obj_second10, obj_second10*60);
1173 | result=log_ptr->num_ok+log_ptr->num_failed;
1174 | }
1175 | return(result);
1176 |
1177 | } /* UD_process_stream */
1178 |