#include #include #include #include "../master/abs_msg.h" #include "private.h" #include "prototypes.h" extern char *device_name; extern char *master_server; #define RETRY_INTERVAL 300 int label_tape_for_dump(dev_handle_t *dh, int jobid); int try_hard_to_write_footer(dev_handle_t *dh, net_volume_t *cvp, int *error_count, int error_max, net_dumpset_t *dp, job_pt jp, abs_label_t *label, dump_params_pt params); int handle_eom(dev_handle_t *dh, job_pt jp, abs_label_t *info, net_dumpset_t *dp, char *verfmode); void set_volume_dump_status(net_volume_t *vp, int status); void mark_tape_log_status(media_t *mp, int threshold); /* * Procedure: do_dump * * Parameters: dh - address of device specific information * dp - address of dumpset information for to be dumped * jp - address of job description info * params - address of job parameters info * * Description: do_dump is the high-level routine which controls the * dump processing for the dumpset represented by "dp". This routine * iterates through all volumes in the dumpset invoking FS_DUMP (filesystem- * specific dump operation) on each volume until all volumes are dumped * successfully or the maximum retry count for each volume is exceeded. * * The master expects the slave to return the following information * after the dump is completed: * - summary of dump statistics (this info is in 2 places * in the "job_stat" structure and in the * dumpstats field of the dumpset. * - status code for each volume dumped (contained in the volume's * "net_volume_t" structure); * * - list of tapes used (returned as part of the dumpset structure) * * - volume to tape mappings; each volume entry will contain informaation * indicating which tape it was successfully written to. This is * to support dumps which consist of multiple tapes. * * * The slave will send a status report to the master before it begins * the dump of a volume. as a result of this status report, the slave * can receive a cancellation request from the master. If the slave * does receive a cancellation request, it will stop the dump at * this time. For canceled jobs, no results are returned to the master. * * On failed volumes, do_dump will attempt to move the device backward * over any data which was written for the failed volume, thus overwriting * the bad data. If the underlying device does not support reverse operation, * the do_dump will add the volume entry to the tape footer block. The tape * footer block is written at the end of the last tape of the dump. */ #define curr_media tape_sets->tapes int do_dump(dev_handle_t *dh, net_dumpset_t *dp, job_pt jp, dump_params_pt params, job_status_pt job_stat) { int code, media_errors, i, rpc_code, fileno; int percent, bad, total, data_written; int abort_job , last_requested=0, scode; net_volume_t *cvp, *retry, *prev, *bad_vols; abs_label_t label; void *handlep; char buf[512]; fs_ops *fsp; struct timeval now; char *datep; float p,t; memset(job_stat, 0, sizeof(job_status_t)); job_stat->total_volumes = dp->nvolumes; if (dp->nvolumes <=0 ) return(ABS_NO_DUMPSET); fsp = get_fs_ops(dp->d_fstype); if (fsp == (fs_ops *) NULL){ set_volume_dump_status(dp->volumes, ABS_INVALID_FSTYPE); return(ABS_INVALID_FSTYPE); } /* * Locate the media item in the specified media device * and validate its internal id with the master. * cycle through will make a best effort to come up with * a tape which we can write on. */ while (( code = cycle_through_tapes(dh, jp->jobid, (char *)NULL, (media_id_pt) NULL, ABS_FREE_MEDIA)) == ABS_INVALID_MEDIA_ID){ /* * Request a mount of something useable */ if (last_requested == 0){ code = request_service(jp->jobid, ABS_TAPE_MOUNT_REQ, NULL); if (code) return(code); last_requested = 1; }else last_requested = 0; sleep(ABS_MOUNT_INTERVAL); } if (code){ if (code != ABS_BLANK_MEDIUM){ set_volume_dump_status(dp->volumes, code); return(code); } code = label_tape_for_dump(dh, jp->jobid); if (code){ set_volume_dump_status(dp->volumes, code); return(code); } } /* * Create Tapeset info for this tape; during the course * of normal operation dp->tapes_sets may have one * or more "tapes" chained off of it. When a media * chage occurs on end-of-medium, the slave executes * a checkpoint, which causes the data for that tape * to be sent to the master. IF the checkpoint fails, * the slave will keep the tape entry and its data around * and send it the next time that a check point occurs * or at the end of this job. * * New media entries are added at the beginning of the list. * * "curr_media" is a shorthand for dp->tape_sets->tapes */ dp->tape_sets = (tapeset_t *) malloc(sizeof(tapeset_t)); if (dp->tape_sets == (tapeset_t *) NULL){ return(ABS_NO_MEM); } memset(dp->tape_sets, 0 , sizeof(tapeset_t)); dp->curr_media = (media_pt)malloc(sizeof(media_t)); if (dp->curr_media == (media_pt)NULL){ free(dp->tape_sets); dp->tape_sets = (tapeset_t *) NULL; return(ABS_NO_MEM); } memset(dp->curr_media, 0, sizeof(media_t)); /* * Read the current label information to get the * static info from the label */ code = SLAVE_REWIND(dh); if (code){ set_volume_dump_status(dp->volumes, code); return(code); } if ((code = read_label(dh, &label))){ set_volume_dump_status(dp->volumes, code); return(code); } /* * save the internal id info for this tape in the tapeset entry */ memcpy(&dp->curr_media->internal, &label.tapeid, sizeof(media_id)); dp->curr_media->log_tape = 1; dp->curr_media->total_vols = 0; dp->curr_media->bad_vols = 0; /* * modify the label fields which change with each dump; */ gettimeofday(&now); label.block_size = dh->bufsize; label.level = params->dump_from; label.uses++; label.sequence = 1; dp->tape_sets->date_dumped = now.tv_sec; label.tape_created = now.tv_sec; strncpy(&label.cell, dp->d_cell, ABS_MAX_DOMAIN); strncpy(&label.dumpset, dp->d_name, ABS_MAX_DNAME); /* * Write the new label to the medium; Also writes an EOF marker. */ code = SLAVE_REWIND(dh); if (code){ set_volume_dump_status(dp->volumes, code); return(code); } if ((code = write_label(dh, &label))){ set_volume_dump_status(dp->volumes, code); return(code); } /* * Start out with all volumes on the retry list. Move them * to dp->curr_media->volumes as they complete successfully. We * stop when the retry list is empty or when we have exceeded * the maximum number of retries defined in the dumpset structure. */ bad_vols = (net_volume_t *) NULL; retry = dp->volumes; dp->volumes = (net_volume_t *) NULL; prev = (net_volume_t *) NULL; bad = media_errors = 0; gettimeofday(&now); datep = ctime(&now.tv_sec); strcpy(dp->dumpstats.start, datep); dp->dumpstats.ntapes = 1; dp->dumpstats.bytes_processed = 0; fileno=1; abort_job = 0; if (dp->d_retry <= 0 ) dp->d_retry = 1; sprintf(buf,"Starting dump of dumpset %s, level %d\n", dp->d_name, dp->dumpstats.level); slave_log_error(ABS_INFORMATIONAL,"do_dump", buf); /* * the big Loop */ for (i = 0; i< dp->d_retry; i++){ if (abort_job) break; for(cvp = retry; cvp != (net_volume_t *)NULL;){ if (abort_job) break; if (i > 0) sleep(RETRY_INTERVAL); /* * Get a connection to this server which hosts the * volume. */ sprintf(buf,"Connecting to Server %s",cvp->v_server); slave_log_error(ABS_INFORMATIONAL, "do_dump", buf); handlep = FS_CONNECT(fsp, cvp->v_server, dp->d_cell, (char *)NULL, &code, ABS_BACKUP_OP); if (handlep == (void *) NULL){ sprintf(buf,"Failed to Connect to %s",cvp->v_server); slave_log_error(ABS_INFORMATIONAL, "do_dump", buf); cvp->v_status = ABS_FS_CONNECT; prev = cvp; cvp = cvp->next; job_stat->failures++; continue; } /* * Send a quick status report to the master telling * it which volume we are about to dump. This is * necessary because dump jobs are quite lengthy and * the Slave isn't threaded, so it can't accept status * queries while the job is running */ strcpy(job_stat->current_volume, cvp->v_name); gettimeofday(&now); job_stat->date = now.tv_sec; sprintf(buf,"Dumping volume %s, retry = %d\n",cvp->v_name, i); slave_log_error(ABS_INFORMATIONAL, "do_dump", buf); code = send_status(jp->jobid, job_stat); if (code == ABS_JOB_CANCELED || get_slave_state() == SLAVE_TERMINATED){ /* * We've been requested to stop our work at the * next convenient point. Here is a good place * to stop; we do not return any results if * we are canceled. */ if (retry != (net_volume_t *) NULL) abs_free_netvol(retry); if (bad_vols != (net_volume_t *) NULL) abs_free_netvol(retry); if (code == ABS_JOB_CANCELED) slave_log_error(ABS_INFORMATIONAL, "do_dump", "Job cancel received"); else{ slave_log_error(ABS_INFORMATIONAL, "do_dump", "slave termination received"); } return(ABS_JOB_CANCELED); } data_written = 0; code = FS_DUMP(fsp, handlep, dh, params->dump_from, jp, cvp, &data_written, &label, params); /* * Sort through the error codes. Those which are * deemed recoverable indicate that the volume should * be retried later (comm failure, call timeout, auth * failures). Those which are unrecoverable will not * be retried (volume does not exist on server, is * damaged or protocol version mismatch). */ sprintf(buf,"Finished volume %s, code = %d\n",cvp->v_name, code); slave_log_error(ABS_INFORMATIONAL, "do_dump", buf); if (code || cvp->v_status){ job_stat->failures++; if (code == ABS_EOM_DETECTED){ /* * Reverse one record and write volume footer * */ scode = SLAVE_NEXT_RECORD(dh, -1); if (scode == 0) write_volume_footer(dh, cvp, code); /* * handle eom will worry about dismounting * the current tape and asking for a new one * It also fills in the label information for * the newly mounted tape; it also executes * a checkpoint which may change the value * of dp->tape_sets->tapes. */ code = handle_eom(dh, jp, &label, dp, params->verf_mode); if (code){ /* * we were unable to properly deal with end of medium */ slave_log_error(ABS_FATAL, "do_dump", "unable to handle End of Medium situation - Dump Aborted"); goto dump_cleanup; } dp->dumpstats.ntapes++; /* * redo this volume - don't advance the cvp pointer */ continue; }else if (code == ABS_MEDIA_ERROR){ /* * We abort if the media error threshold has been * exceeded. If the threshold is not exceeded * Then a footer will be written by the code * following this if statement. */ media_errors++; slave_log_error(ABS_ERROR,"do_dump","Media errors on device"); if (media_errors > params->media_errors){ /* * This tape is bad. Set "log_tape" to false * so the info for this tape does not get * logged in the database */ dp->curr_media->log_tape = 0; goto dump_error; } }else if (UNRECOVERABLE(code)){ /* * Remove from retry queue and * place volume on bad list */ bad++; if (prev){ prev->next = cvp->next; cvp->next = bad_vols; bad_vols = cvp; cvp = prev->next; } else{ retry=retry->next; cvp->next = bad_vols; bad_vols = cvp; cvp = retry; } job_stat->failures++; code = try_hard_to_write_footer(dh, cvp, &media_errors, params->media_errors, dp, jp, &label, params); if (code != 0 && code != ABS_EOM_DETECTED){ slave_log_error(ABS_ERROR, "do_dump", "Dump aborted due to media/device errors"); goto dump_cleanup; } continue; } /* else EOM */ /* * This volume will be retried later */ prev = cvp; cvp = cvp->next; if (data_written){ dp->curr_media->bad_vols++; /* * If something was written to the tape, try to backup * to the previous file mark. If the devices does not * support this type of operation, then create a bad * volume entry in the volume summary table. */ #ifdef REVERSE code = SLAVE_NEXT_FILE(dh, -1); if (code){ fileno++; } #else /* * Write a volume footer indicating that this volume * is bad */ code = try_hard_to_write_footer(dh, cvp, &media_errors, params->media_errors, dp, jp, &label, params); if (code != 0 && code != ABS_EOM_DETECTED){ slave_log_error(ABS_ERROR, "do_dump", "Dump aborted due to media/device errors"); goto dump_cleanup; } #endif } continue; } /* if code */ /* * we are here because we have successfully dumped the * volume. Increment the total dumped and remove the * the volume from the retry queue */ code = try_hard_to_write_footer(dh, cvp, &media_errors, params->media_errors, dp, jp, &label, params); if (code != 0 && code != ABS_EOM_DETECTED){ slave_log_error(ABS_ERROR, "do_dump", "Dump aborted due to media/device errors"); goto dump_cleanup; } job_stat->vols_processed++; dp->curr_media->total_vols++; /* * The volume must be re-dumped since all of its * data, including the footer must fit on the same tape * Do not advance the cvp pointer. */ job_stat->bytes_processed += cvp->v_size; cvp->position = fileno; fileno++; if (prev){ prev->next = cvp->next; cvp->next = dp->curr_media->volumes; dp->curr_media->volumes = cvp; cvp = prev->next; } else{ retry=retry->next; cvp->next = dp->curr_media->volumes; dp->curr_media->volumes = cvp; cvp = retry; } } /* for cvp */ /* * no more volumes on the retry list - we are done */ if (retry == (net_volume_t *) NULL) break; } /* for retry */ dump_cleanup: mark_tape_log_status(dp->curr_media, dp->d_failthresh); /* * count the volumes which could not be dumped in "dp-d_retry" times * and add it to the total bad volume count */ if (retry != (net_volume_t *) NULL){ for( cvp = retry; cvp->next != (net_volume_t *) NULL; bad++, cvp = cvp->next); } if (bad_vols != (net_volume_t *) NULL){ for( cvp = bad_vols; cvp->next != (net_volume_t *) NULL; cvp = cvp->next); /* * Add the bad volumes back to the dumpset volume chain */ cvp->next = dp->volumes; dp->volumes = bad_vols; } if (bad > 0 ){ /* * Determine whether the fail threshold has been exceeded for this * job */ t = (float) job_stat->total_volumes; p = (float) bad; percent = (int)((p/t) * 100.0); if (percent > dp->d_failthresh){ code = ABS_ERROR_THRESH; } } dp->dumpstats.bytes_processed = job_stat->bytes_processed; dump_error: sprintf(buf,"Finished dump of dumpset %s, status %d - %d of %d volumes dumped\n", dp->d_name, code, job_stat->vols_processed, dp->nvolumes); slave_log_error(ABS_INFORMATIONAL, "do_dump", buf); gettimeofday(&now); datep = ctime(&now.tv_sec); strcpy(dp->dumpstats.stop, datep); return(code); } /* * Cycle through all the media items on the device looking * for the specified item * Search by (for devices that support * scanning the external label) * - medium with matching internal id * - any medium with valid abs label * - any medium * * */ int cycle_through_tapes(dev_handle_t *dh, int jobid, char *extid, media_id_pt mip, int valid) { int code, rpc_code; char buf[512]; abs_label_t abs_label; dev_table_t *dp = dh->dev_ops; /* * Find a tape with matching the caller's selection critera. * This can be a match by id, by * internal id only, or by any tape with valid abs label or * any tape * */ code = 1; while (code != 0){ /* * loop through allof the media on the device. For * single cartrige tape systems this funciton is a no-op * for multi-cartridge systems, the device dependent routine * is responsible for informing the master if the a media * mount is required */ code = SLAVE_NEXT_MEDIA(dh, extid, mip, valid); if (code){ /* fatal, unrecoverable error */ sprintf(buf, "Media/Device errors from device %s \n", device_name); slave_log_error(ABS_FATAL,"do_dump", buf); return(code); } } if (valid != 0){ /* * For valid !=0 we must validate the tape with the * master. First we call validate_label telling it that we * are expecting a tape with a valid abs label. * validate_label will try to read the tape and determine * whether the tape meets our expectations (has a valid label). * A null media id is specified when the caller is not looking * for a specific tape; i.e., any labeled tape will do. */ if (mip != (media_id_pt *) NULL){ code = validate_label(dh, mip, ABS_VALID_LABELID, &abs_label); /* * This is a restore request, so we don't need to validate the * tape with the master * * If there isn't any tape in the drive, sometimes we get * "BLANK_MEDIUM" - map this to INVALID_MEDI_ID to force a * media mount request to happen. */ if (code == ABS_BLANK_MEDIUM) return(ABS_INVALID_MEDIA_ID); if (code) return(code); }else{ code = validate_label(dh, (media_id_pt)NULL, ABS_VALID_LABEL, &abs_label); if (code) return(code); /* * We now have a tape whose label appears to be valid. * Request a tape status query from the master to see what state * this tape is in (e.g. VALID, FREE). "code" will actually contain * the tape status inforamtion. */ rpc_code = slave_validate(&abs_label.tapeid, jobid, &code); if (rpc_code) return(rpc_code); if (code == ABS_JOB_CANCELED) return(code); if (code == ABS_INVALID_JOB) return(code); /* * "valid" contains the state which the caller requests (e.g. needs * a free tape, etc). "code " contains the tape's state as * listed in the abs database. */ if (valid != code){ sprintf(buf,"Media validation with master fails, media state = %d, expected %d\n", code, valid); slave_log_error(ABS_WARNING, "cycle_through_tapes", buf); return(ABS_INVALID_MEDIA_ID); } code = 0; } } /* * We are here because the medium we wanted is mounted so * Rewind it. */ code = SLAVE_REWIND(dh); if (code){ sprintf(buf, "device %s Rewind failure \n", device_name); slave_log_error(ABS_FATAL,"do_dump", buf); } return(code); } /* * Procedure: perform_media_verification * * Parameters: -dh address of device information * dp - address of dumpset information * verf_mode - verification mode to use * job_stat - address of job status information * jobid - job id assigned by master * * Description: This function closes the media, causing an EOM marker * to be written at the end of the dump data and rewinds the tape. The * tape is then re-opened for read only access. We send a status report * to the master informing it that we are now performing media verification. * We then proceed to call the appropriate media verification function */ int perform_media_verification(dev_handle_t *dh, net_dumpset_t *dp, char *verf_mode, job_status_t *job_stat, int jobid) { int code; abs_label_t label; /* * All modes of verification verify the label, so let's * do it here. */ if (!strcmp(verf_mode, "NONE")) return(0); /* * Tell Master we are verifying the tape */ strcpy(job_stat->current_volume, "VERIFY MEDIA"); code = send_status(jobid, job_stat); code = SLAVE_REWIND(dh); code = SLAVE_CLOSE_PARTIAL(dh); code = SLAVE_OPEN(dh->dev_ops, &dh, dh->devname, O_RDONLY, jobid); if (code) return(code); code = read_label(dh, &label); if (code) return(code); if (!strcmp(verf_mode, "FULL")){ return(slave_full_verification(dh, dp)); }else if (!strcmp(verf_mode, "QUICK")){ return(slave_quick_verification(dh, dp)); }else { return(ABS_INVALID_VERF); } } /* * Procedure: try_hard_to_write_footer * * Parameters: dh - address of device information. * cvp - address of volume information * error_count - address of current media error count * error_max - media error threshold * dp - address of dumpset information * jp - address of job information. * label - address to write label inforamtion. * * Description: This procedure attempts to write a volume footer * using the inforamtion provided by cvp. It will repeatedly attemp * to write the footer until it is successful or the media error threshold * is exceeded, or it detects EOM. IF EOM is detected, it will request * a mount of a new tape and return an error code indicating so. * * In the case of EOM, we do not attempt to write a footer on the new * tape since volumes are not allowed to span tapes. * * Modifies: label, only if a new tape is mounted * dp->tapesets , only if a new tape is mounted * * Memory allocation: memory allocated to dp will be freed by the RPC * when the dump service routine completes. */ int try_hard_to_write_footer(dev_handle_t *dh, net_volume_t *cvp, int *error_count, int error_max, net_dumpset_t *dp, job_pt jp, abs_label_t *label, dump_params_pt params) { int code; media_t *mp; do { code = write_volume_footer(dh, cvp, cvp->v_status); if (code == ABS_MEDIA_ERROR) *error_count = *error_count +1; }while(code == ABS_MEDIA_ERROR && *error_count <= error_max); if (code == 0) return(code); if (code == ABS_MEDIA_ERROR){ slave_log_error(ABS_ERROR,"try_hard_to_write_footer", "Media errors on device"); return(code); } if (code == ABS_EOM_DETECTED){ code = handle_eom(dh, jp, label, dp, params->verf_mode); if (code){ /* * we were unable to properly deal with end of medium */ slave_log_error(ABS_FATAL, "try_hard_to_write_footer", "unable to handle End of Medium situation - Dump Aborted"); return(code); }else{ dp->dumpstats.ntapes++; /* * So that the caller knows the footer could not * be written on the same tape; */ return(ABS_EOM_DETECTED); } } return(code); } /* * Procedure: label_tape_for_dump * * Parameters: dh -address of device information * jobid - job id assigned by master for this job * * Description: This functionr equ3ests the master to generate * an internal label identifier and proceeds to label the tape * using that internal identifier. */ int label_tape_for_dump(dev_handle_t *dh, int jobid) { media_id_t mip; int code; /* * Get an intenal label from the master */ code = request_internal_label(jobid, &mip); if (code) return(code); code = slave_label_medium(NULL, dh, NULL, &mip); return(code); } /* * Procedure: set_volume_dump_status * * Parameters: vp - address of linked list of volume structures * status - status code * * Description: This function sets the status code of each * volume in the linked list to "status" * * Modifies: "v_status" field of each volume in list */ void set_volume_dump_status(net_volume_t *vp, int status) { net_volume_t *cvp; for (cvp = vp; cvp != (net_volume_t *)NULL; cvp = cvp->next){ cvp->v_status = status; } } void mark_tape_log_status(media_t *mp, int threshold) { float t, b; int percent; t = (float) mp->total_vols; b = (float) mp->bad_vols; percent = (int)((b/t) * 100.0); if (percent > threshold){ mp->log_tape = 0; } }