/* ** This software is Copyright (c) 1989, 1990, 1991 by Kent Landfield. ** ** Permission is hereby granted to copy, distribute or otherwise ** use any part of this package as long as you do not try to make ** money from it or pretend that you wrote it. This copyright ** notice must be maintained in any copy made. ** */ #if !defined(lint) && !defined(SABER) static char SID[] = "@(#)news_arc.c 2.3 5/9/91"; #endif #include #include #include #include #include #include "article.h" #include "cfg.h" /* ** Defines for the type of "problems" ** encountered in saving the articles. */ #define DUP_PROB 0 #define NAME_PROB 1 #define VOL_PROB 2 #define TYPE_PROB 3 #define CHECKHASH_PROB 4 #define MSNG_HASH_PROB 5 #define CA_NAME_PROB 6 #define EXTERNAL_PROB 7 int test = 0; int inum = 0; int problem_article; extern struct group_archive *newsgrp; extern int overwrite, retrieve; extern char nntp_tmp_path[]; int fclose(); int stat(); int unlink(); int do_checkhash(); int remove_suffix(); char *strchr(); char *strcpy(); char *strcat(); char *do_problem(); char *basename(); char *suffix(); char *expand_name(); char *copy_article(); FILE *efopen(); void init_article(); void store_line(); void dump_article(); void record_problem(); void write_patch_log(); void get_header(filename) char *filename; { char *dp; int header_ok = 0; int last = TEXT; FILE *gfp; init_article(); gfp = efopen(filename,"r"); (void) strcpy(article.newsarticle, filename); while (fgets(s,sizeof s,gfp) != NULL) { if (debug) (void) fprintf(logfp, "BUF = [%s]",s); /* ignore continuation lines for now */ if (*s == ' ' || *s == '\t') continue; if (!isalpha(*s) || (strchr(s,':') == NULL)) { header_ok++; if (header_ok >= 2) { if (*s == '\n' && last == BLANK) continue; break; } if (*s == '\n') last = BLANK; else last = TEXT; continue; } last = TEXT; dp = s; while (*++dp) if (*dp == '\n') *dp = '\0'; store_line(); } (void) fclose(gfp); if (debug) dump_article(); } #ifdef REDUCE_HEADERS struct hdrstokeep { char *ststr; int stbytes; }; struct hdrstokeep hdrs[] = { { "From:", (sizeof "From:") }, { "Newsgroups:", (sizeof "Newsgroups:") }, { "Subject:", (sizeof "Subject:") }, { "Message-ID:", (sizeof "Message-ID:") }, { "Date:", (sizeof "Date:") }, { "Approved:", (sizeof "Approved:") }, { "X-Checksum-Snefru:", (sizeof "X-Checksum-Snefru:") }, { "X-Md4-Signature:", (sizeof "X-Md4-Signature:") }, { NULL, 0 }, }; int keep_line(argstr) char *argstr; { int strncmp(); struct hdrstokeep *pt; pt = &hdrs[0]; while ((pt->ststr) != NULL) { if (strncmp(argstr, pt->ststr, (pt->stbytes-1)) == 0) return(TRUE); pt++; } return(FALSE); } #endif /* REDUCE_HEADERS */ int copy(source, target) char *source, *target; { char *strchr(); int fputs(); FILE *from, *to; char fbuf[BUFSIZ]; #ifdef REDUCE_HEADERS int inheader; inheader = TRUE; #endif /* REDUCE_HEADERS */ if (verbose) { (void) fprintf(logfp,"archive <%s> to <%s>\n",source,target); if (test) return(0); } if ((from = fopen(source, "r")) == NULL) { (void) fprintf(errfp,"%s: cannot open %s\n",progname,source); return (-1); } if ((to = fopen(target, "w")) == NULL) { (void) fclose(from); (void) fprintf(errfp,"%s: cannot create %s\n",progname,target); return (-1); } while (fgets(fbuf, BUFSIZ, from) != NULL) { #ifdef REDUCE_HEADERS /* ** Read the source and do not print any headers ** unless specified in the "keep" headers table. */ if (inheader) { /* ** Have I encountered a line without a line type ? */ if (!isalpha(*fbuf) || (strchr(fbuf,':') == NULL)) inheader = FALSE; else { /* ** Determine the type of the header line and ** decide if this is a line to be kept or pitched. */ if (!keep_line(fbuf)) continue; } } #endif /* REDUCE_HEADERS */ if (fputs(fbuf, to) == EOF) { (void) unlink(target); (void) fclose(from); (void) fclose(to); (void) fprintf(errfp,"%s: bad copy to %s\n",progname,target); return (-1); } } (void) fclose(from); (void) fclose(to); return(0); } /* ** mkparents: ** ** If any parent directories in ** fullname don't exist, create them. */ int mkparents(fullname) char *fullname; { int access(); int makedir(); char *strrchr(); register char *p; char b[MAXNAMLEN]; int rc; (void) strcpy(b, fullname); if ((p = strrchr(b, '/')) != NULL) *p = '\0'; else /* no directories in fullname */ return(0); if (*b == '\0') /* are we at the root ? */ return(0); if (access(b, 0) == 0) return(0); if (mkparents(b) == -1) return(-1); if ((rc = makedir(b, DIR_MODE, newsgrp->owner, newsgrp->group)) != 0) record_problem("makedir failed attempting to make %O", b, newsgrp); return(rc); } char *copy_article(ng, filename,path) struct group_archive *ng; char *filename; char *path; { void write_archived(); if (copy(filename,path) != 0) { (void) fprintf(errfp,"copy failed for %s to %s\n",filename,path); return(NULL); } /* ** Write the message-id to the .archived file in the newsgroup's ** BASEDIR directory since we do not want it rearchived tomorrow. */ write_archived(header.ident, path); /* ** Check if the file is a patch. If so, log ** the patch information into the patch log ** in a *non-configurable* format so that ** applications can be written to access the ** file's "known format". */ if (article.rectype == PATCH) write_patch_log(ng,path); /* ** Return the path to the archived file. */ return(path); } char *save_article (filename,ng) char *filename; struct group_archive *ng; { char *format_output(); void check_archive_name(); void chronpath(); char *final_path; char *kp; char cmdline[BUFSIZ]; char command[BUFSIZ]; static char path[MAXNAMLEN]; struct stat sb; problem_article = FALSE; path[0] = '\0'; /* ** The news article has been read and the header ** information is filled into the appropriate ** data structures. */ /* ** If the MD4 or Snefru headers exists, check the article's ** checksum for validity. */ if (header.x_checksum[0]) { if (*(ng->checkhash)) { if (do_checkhash(ng->checkhash, filename) != 0) return(do_problem(CHECKHASH_PROB,ng,filename,path)); } else if(*checkhash) { if (do_checkhash(checkhash, filename) != 0) return(do_problem(CHECKHASH_PROB,ng,filename,path)); } } else { if (*(ng->checkhash) || *checkhash) return(do_problem(MSNG_HASH_PROB,ng,filename,path)); } /* ** Build the path string for the final resting spot ** for the new archive member. */ switch(ng->type) { case ARCHIVE_NAME: /* ** The header's archive_name contains the filename in ** an "elm/part06" format. */ if ((article.volume == -1) || (!header.archive_name[0])) { /* ** If the posting is an administration (ADM) posting ** then create a valid Archive-Name:. In other words, ** cheat bigtime... c.s.apple2 uses these headers and does ** not put in any auxiliary headers. They should be pitched ** in that newsgroup, according to Jonathan, but I like to ** keep a copy of everything that goes through the newsgroup. ** The history thing ya know.. :-) */ if (article.rectype == ADMINISTRATION) (void) sprintf(header.archive_name,".admin/ADM%d",article.issue); else return(do_problem(NAME_PROB, ng,filename,path)); } /* ** Assure the address is relative and ** that some prankster can not do nasty ** things to your system files by having ** an Archive-name line like: ** ../../../../../etc/passwd */ check_archive_name(header.archive_name); /* ** Check to see if the article is a patch. If so, ** check to see if the administrator wishes to ** store the patch with the initially posted ** articles. This really relys on the archive name ** being correct. */ if (article.rectype == PATCH && ng->patch_type == PACKAGE) /* ** Store the patch in the volume specified with the ** Archive-name: specified file name. */ #ifdef ZEROFILL (void) sprintf(path,"%s/%s%.02d/%s", ng->location, VOLUME, #else (void) sprintf(path,"%s/%s%d/%s", ng->location, VOLUME, #endif article.patch_volume, header.archive_name); else #ifdef ZEROFILL (void) sprintf(path,"%s/%s%.02d/%s", ng->location, VOLUME, #else (void) sprintf(path,"%s/%s%d/%s", ng->location, VOLUME, #endif article.volume, header.archive_name); break; case VOLUME_ISSUE: /* ** The article filename contains the filename in ** a "v01i001" format. */ if ((article.volume == -1) || (!article.filename[0])) return(do_problem(VOL_PROB,ng,filename,path)); #ifdef ZEROFILL (void) sprintf(path,"%s/%s%.02d/%s", ng->location, VOLUME, #else (void) sprintf(path,"%s/%s%d/%s", ng->location, VOLUME, #endif article.volume, article.filename); break; case ARTICLE_NUMBER: /* ** Store in same filename - thanks news... */ (void) sprintf(path,"%s/%s", ng->location, filename); break; case CHRONOLOGICAL: /* ** The chronpath() is called to create a path for an article ** to be stored in chronological ordering. We need to be sure ** that the issue number is not in use. This is necessary to ** handle multiple runs of the program on the same day. ** ** The idea here is to have the program check to see if the ** issue number to be used is available. ** There should be no duplicates here ever... :-) ** [ just don't blow away your .archived file... :-( ] ** Need to assure that the compression suffix in not attached ** as well since the prior run that day may use compression. */ do { do { ++inum; chronpath(ng->location, path, inum); } while (stat(path ,&sb) == 0); /* ** expand the path to the file to include the ** compression suffix if necessary. */ final_path = expand_name(path, ng); } while (stat(final_path ,&sb) == 0); break; case ONLY_ARCHIVE_NAME: /* ** This is for searching for articles with an Archive-name: ** no other information. The header's archive_name contains ** the filename in an "elm/part06" format. */ if (!header.archive_name[0]) return(NULL); /* ** Assure the address is relative and that some prankster can ** not do nasty things to your system files by having an ** Archive-name line like: ** ../../../../../etc/passwd */ check_archive_name(header.archive_name); (void) sprintf(path,"%s/%s",ng->location,header.archive_name); break; case COMP_ARCHIVES: /* ** Comp.archives supplies an Archive-name: but no volume ** so it has to be treated as an separate type of archiving. ** It could have been squeezed into ARCHIVE-NAME but that ** does not allow for easy changes in the future for either. ** ** The header's archive_name contains the filename in ** an "elm/part06" format. */ if (!header.archive_name[0]) return(do_problem(CA_NAME_PROB, ng,filename,path)); /* ** Assure the address is relative and ** that some prankster can not do nasty ** things to your system files by having ** an Archive-name line like: ** ../../../../../etc/passwd */ check_archive_name(header.archive_name); (void) sprintf(path,"%s/%s",ng->location,header.archive_name); break; case EXTERNAL_COMMAND: /* ** This type of archiving is being handled by a script/application ** outside of the rkive program. In this situation, rkive is being ** used to locate the articles to be archived, pipe the information ** about the article to the external command to process and wait for ** a success/failure status to be returned. At that point, rkive ** logs the status. Compression is bypassed regardless of whether or ** not it is specified. Index records are written with the info as ** specified by the format. This means that the external command ** will have to know about duplicates, reposts and compression. The ** execution command string will be expanded to see if the external ** command specified in the rkive.cf file has any parameters to be ** filled in... */ /* ** Need to build a path to the archive location if there was one. ** Just using Article-Number for the file name. If there is a better ** idea here, I am all ears... */ (void) sprintf(path,"%s/%s", ng->location, filename); /* ** Make any required parent directories along the way. ** Done so that the external applications will have a ** base directory in which to perform there "thing". */ if (mkparents(path) == -1) return(NULL); /* ** Expand and build the command line to execute. ** The file to be archived will always be the first command line ** argument regardless of what is supplied in the ARCHIVE_CMD line. ** ** First separate the command from the options. ** If a space is found after the cmdline is striped put a ** null there and then replace it with a space after the check... ** Rebuild the command string. */ if (*ng->arch_command) (void) strcpy(command, ng->arch_command); else if (*arch_command) (void) strcpy(command, arch_command); else return(do_problem(EXTERNAL_PROB,ng,filename,path)); if ((kp = strchr(command,' ')) != NULL) *kp = '\0'; #ifdef NNTP if (retrieve == FROM_NNTP) { (void) sprintf(cmdline, "%s %s", command, nntp_tmp_path); } else { #endif (void) sprintf(cmdline,"%s %s/%s/%s", command, spooldir, newsgrp->ng_path, filename); #ifdef NNTP } #endif if (kp != NULL) { /* There are options to be stored... */ *kp = ' '; (void) strcat(cmdline,kp); } kp = format_output(cmdline, filename, ARCHIVE); /* ** Execute the command and wait for a completion status. ** If a problem exists then alert the administrator to the problem */ if (verbose) (void) fprintf(logfp,"executing <%s>\n",kp); if (!test) { if (system(kp) != 0) return(do_problem(EXTERNAL_PROB,ng,filename,path)); } /* ** Write the message-id to the .archived file in the newsgroup's ** BASEDIR directory since we do not want it rearchived tomorrow. */ write_archived(header.ident, path); /* ** Check if the file is a patch. If so, log ** the patch information into the patch log ** in a *non-configurable* format so that ** applications can be written to access the ** file's "known format". */ if (article.rectype == PATCH) write_patch_log(ng,path); /* ** Return the path to the archived file. */ return(path); default: /* ** We have got problems.... */ return(do_problem(TYPE_PROB,ng,filename,path)); } #ifdef ADD_REPOST_SUFFIX if (article.repost == TRUE) /* ** The ADD_REPOST_SUFFIX code adds the REPOST_SUFFIX ** to any file that has been indicated as a repost ** by the moderator. This should not be used with ** Archive-Name archiving on a filesystem with 14 ** character filename limits or filename truncation ** can occur. You have been warned... :-( ** ** After adding the REPOST_SUFFIX, the filename is ** treated as any other file with the duplication ** checks and all... */ (void) strcat(path,REPOST_SUFFIX); #endif /* ADD_REPOST_SUFFIX */ /* ** expand the path to the file to include the ** compression suffix if necessary. */ final_path = expand_name(path, ng); /* ** Make any necessary directories ** along the way. */ if (mkparents(path) == -1) return(NULL); /* ** Check to assure that there is not already ** a file with the same file name. If so ** copy (or archive) the file to the problems ** directory. ** ** This works for REPOSTS as well. ** If the REPOST arrives and there is ** no file currently at the archive location, the ** REPOST is installed in the correct archive ** location. ** If there is a file that exists when a REPOST ** arrives, the REPOST is then handled in do_problem(). */ if ((stat(final_path ,&sb) == 0) && !overwrite) /* duplicate found */ return(do_problem(DUP_PROB,ng, filename, final_path)); return(copy_article(ng, filename, path)); } char *do_problem(type_of_problem, ng, file, path) int type_of_problem; struct group_archive *ng; char *file; char *path; { void set_ownership(); #ifdef MV_ORIGINAL char crnt_path[MAXNAMLEN]; #endif /*MV_ORIGINAL */ char pmess[BUFSIZ]; char *final_path; int nm; struct stat sb; problem_article = TRUE; /* ALERT THE ADMINISTRATOR THAT A PROBLEM WAS ENCOUNTERED ** ** A problem has been encountered. It could be that there is an ** format mismatch or there is already a file with the same ** issue/archive/msg-id name. ** Copy the problem file to the problems directory. ** Alert the Administrator that a problem was received. */ (void) sprintf(pmess,"PROBLEM: Article %s in %s ",file,ng->ng_name); switch( type_of_problem ) { case NAME_PROB: (void) strcat(pmess,"does not support Archive-Name Archiving.\n"); break; case VOL_PROB: (void) strcat(pmess,"does not support Volume-Issue Archiving.\n"); break; case TYPE_PROB: (void) strcat(pmess,"has an invalid archive TYPE specified.\n"); break; case CHECKHASH_PROB: (void) strcat(pmess,"failed article checksum verification test.\n"); break; case MSNG_HASH_PROB: (void) strcat(pmess,"is missing expected article checksum verifications header.\n"); break; case DUP_PROB: if (article.repost != TRUE) (void) strcat(pmess,"is a Duplicate article.\n"); else (void) strcat(pmess,"is a Reposted article.\n"); (void) sprintf(pmess,"%s\tExisting Archived path - %s", pmess,path); break; case CA_NAME_PROB: (void)strcat(pmess,"does not have a valid Archive-name specified.\n"); break; case EXTERNAL_PROB: (void)strcat(pmess,"System of External command returned Non-zero value...\n"); } /* print the message out to the screen, crontab output, etc */ (void) fprintf(errfp,"%s\n",pmess); /* log the initial detection message. */ record_problem(pmess, file, ng); /* Handling Repostings. ** ** MV_ORIGINAL ** The original article is placed into a "original" directory in ** the problems directory (if duplicated). The inbound reposted ** article is placed into the archive in the correct position. ** ** ADD_REPOST_SUFFIX ** If ADD_REPOST_SUFFIX is defined, all reposts will have the ** string specified in REPOST_SUFFIX appended to the archive ** filename so that a repost of elm/part07 would appear in ** the archive as elm/part07-repost prior to any compression. ** The addition of the suffix was done in save_article(). ** Handle this as the true duplicated article that it is. ** ** No Reposting Defines specified: ** The inbound article would be placed into the archive in the ** correct position only if the initial article is not in the archive. ** Otherwise the reposted article is placed in the problems directory ** as a normal duplicate article as it is now. */ #ifdef MV_ORIGINAL if ((article.repost == TRUE) && (ng->type != EXTERNAL_COMMAND)) { /* ** save the duplicated path ** Caution: may have compression suffix attached */ (void) strcpy(crnt_path, path); /* create the storage path for original copy */ /* no slash needed between Originals and crnt_path below.. */ (void) sprintf(path,"%s/%s%s",problems_dir,"Originals",crnt_path); /* Display and record the actions */ (void) sprintf(pmess,"\tMoving %s (original)\n\tto %s",crnt_path,path); (void) fprintf(errfp,"%s\n",pmess); record_problem(pmess, file, ng); /* Make any necessary directories along the way. */ if (mkparents(path) == -1) return(NULL); /* copy the original out of the way */ if (copy(crnt_path,path) != 0) { (void) fprintf(errfp,"copy failed for %s to %s\n", crnt_path, path); return(NULL); } set_ownership(path, path, ng); /* restore the destination path for inbound article */ (void) strcpy(path,crnt_path); /* remove the existing file */ (void) unlink(path); /* ** Must assure that "path" does not have a .Z type ** of suffix used in compression. If it does, it must ** be removed before continuing. This is cheating and ** will probably break but what the hell. */ (void) remove_suffix(path); } else #endif /* MV_ORIGINAL */ /* ** Build the path string for the location of the article in ** the problems directory. Place the file in the appropriate ** directory in Article-Number format. First check to see if ** that Article-Number named file does not exist. This extra ** check is being done to assure that no conflict exists for ** NNTP sites. In this manner problems will be assured to ** be stored as uniquely named, separate files. */ (void) sprintf(path,"%s/%s/%s",problems_dir,ng->ng_path,file); nm = 1; do { final_path = expand_name(path, ng); /* ** Let's assure that neither the uncompressed or ** the compressed version of the article number ** exists where we want to put this problem. ** Don't need to create a different problem... */ if ((stat(path, &sb) == 0) || (stat(final_path, &sb) == 0)) { (void) sprintf(path,"%s/%s/%s.%d",problems_dir,ng->ng_path,file,nm); ++nm; } else nm = 0; } while (nm != 0); /* Display and record the actions */ (void) sprintf(pmess,"\tStoring Article %s at %s\n", file, path); (void) fprintf(errfp,"%s\n",pmess); record_problem(pmess, file, ng); /* Make any necessary directories along the way. */ if (mkparents(path) == -1) return(NULL); return(copy_article(ng, file, path)); } #ifndef NO_MONTH_DIR static char *month[] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; #endif /* NO_MONTH_DIR */ void chronpath(dirloc, path, seqnum) char *dirloc; char *path; int seqnum; { long time(); struct tm *localtime(); long clk; struct tm *crnt; static struct tm now; static int no_time = 1; if (no_time) { clk = time((long *)0); crnt = localtime(&clk); no_time = 0; now = *crnt; } #ifdef NO_MONTH_DIR /* ** Format: ** /usenet/alt/sources/volume89/890629.01 */ # ifdef ZEROFILL (void) sprintf(path,"%s/%s%.02d/%.02d%.02d%.02d.%.02d",dirloc, # else (void) sprintf(path,"%s/%s%d/%.02d%.02d%.02d.%.02d",dirloc, # endif VOLUME, now.tm_year, now.tm_year,now.tm_mon+1,now.tm_mday,seqnum); #else /*!NO_MONTH_DIR*/ /* ** Format: ** /usenet/alt/sources/volume89/Jun/890629.01 */ # ifdef ZEROFILL (void) sprintf(path,"%s/%s%.02d/%s/%.02d%.02d%.02d.%.02d",dirloc, # else (void) sprintf(path,"%s/%s%d/%s/%.02d%.02d%.02d.%.02d",dirloc, # endif VOLUME, now.tm_year, month[now.tm_mon], now.tm_year,now.tm_mon+1,now.tm_mday,seqnum); #endif /* NO_MONTH_DIR */ article.volume = now.tm_year; } void write_patch_log(ng, path) struct group_archive *ng; char *path; { int strlen(); char *sp; FILE *plfp; struct stat sb; if (test) return; /* ** The .patchlog file is used to record the ** information specific to patches that come ** through the newsgroup. ** ** The format of the .patchlog file is: ** # ** # Patch log for comp.sources.whatever ** # ** # Path To Patch Package Initial ** # Patchfile Volume Issue Name Volume Issue ** # ** volume4/conquer4/Part04 6 86 conquer4 4 42-49 ** volume4/conquer4/Part06 6 88 tests3 4 42,47,51 ** volume6/conquer4/Part07 6 89 Unknown 6 89 */ /* ** If this is the first time that an entry is written to the ** patch log, add a header on top of the file for informational ** purposes only... Output in the patch log is not intended ** to be centered under these columns... :-) I'm laaaazy. */ if ((stat(ng->patchlog ,&sb) != 0)) { if (mkparents(path) == -1) return; /* unable to build parent directories */ plfp = efopen(ng->patchlog,"a+"); (void) fprintf(plfp,"#\n#\tPatch log for %s\n#\n", ng->ng_name); (void) fprintf(plfp,"# %-27s%-12s%-15s%-s\n", "Path To", "Patch", "Package", "Initial"); (void) fprintf(plfp,"# %-22s%-18s%-10s%-15s\n#\n", "Patchfile", "Volume Issue", "Name", "Volume Issue"); (void) fclose(plfp); } /* ** Get rid of the base directory. */ sp = path + (strlen(ng->location)+1); plfp = efopen(ng->patchlog,"a+"); (void) fprintf(plfp,"%-25s %3d %5d %-14s %3d %s\n", sp, article.volume, article.issue, article.package_name, article.patch_volume, article.patch_issue); (void) fclose(plfp); return; } int do_checkhash(hashit,filename) char *hashit; char *filename; { int system(); char *kp; char *comp_cmd; char cmd[BUFSIZ]; /* ** get the basename of the command to use. ** strip off possible arguments. */ (void) strcpy(cmd, hashit); if ((kp = strchr(cmd,' ')) != NULL) *kp = '\0'; else if ((kp = strchr(cmd,'\t')) != NULL) *kp = '\0'; comp_cmd = basename(cmd); if (verbose) (void) fprintf(logfp,"%s %s\n", comp_cmd, filename); /* ** build command to execute. */ (void) sprintf(cmd,"%s %s", hashit, filename); if (!test) return(system(cmd) >> 8); else return(0); }