tar._pike
Go to the documentation of this file.
1 /* Copyright (C) 2000-2005 Thomas Bopp, Thorsten Hampel, Ludger Merkens
2  *
3  * This program is free software; you can redistribute it and/or modify
4  * it under the terms of the GNU General Public License as published by
5  * the Free Software Foundation; either version 2 of the License, or
6  * (at your option) any later version.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11  * GNU General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program; if not, write to the Free Software
15  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
16  *
17  * $Id: tar.pike,v 1.1 2008/03/31 13:39:57 exodusd Exp $
18  */
19 inherit "/kernel/module.pike";
20  inherit Filesystem.Base;
21  inherit "/kernel/DocFile";
22  inherit Filesystem.Stat;
23  inherit Filesystem.System;
24  inherit _TarFS;
25 #include <macros.h>
26 #include <database.h>
27 #include <classes.h>
28 //! This is the tar module. It is able to create a string-archive of
29 //! all tared files.
30 class tar : public module.pike,DocFile{
31 public:
32 
33 
34 
35 
36 
37 
38 //#define TAR_DEBUG 1
39 
40 #ifdef TAR_DEBUG
41 #define LOG_TAR(s, args...) werror("tar: "+s+"\n", args)
42 #else
43 #define LOG_TAR(s, args...)
44 #endif
45 
46 #define BLOCKSIZE 512
47 
48 /**
49  * Convert an integer value to oct.
50  *
51  * @param int val - the value to convert
52  * @param int size - the size of the resulting buffer
53  * @return the resulting string buffer
54  */
55 protected:
56  string to_oct(int val, int size)
57 {
58  string v = (string) val;
59  string oct = "";
60  int i;
61 
62 # define MAX_OCTAL_VAL_WITH_DIGITS(digits) (1 << ((digits) * 3) - 1)
63 
64  for ( i = 0; i < size; i++ ) oct += " ";
65 
66  if ( val <= MAX_OCTAL_VAL_WITH_DIGITS(size-1) )
67  oct[--i] = '\0';
68 
69  while ( i >= 0 && val != 0 ) {
70  oct[--i] = '0' + (int)(val&7);
71  val >>=3;
72  }
73 
74  while ( i!=0 )
75  oct[--i] = '0';
76  return oct;
77 }
78 
79 public:
80 
81  private string header;
82 
83 /**
84  * Copy a source string to the header at position 'pos'.
85  *
86  * @param string source - source string to copy.
87  * @param int pos - the position to copy it in 'header'.
88  */
89 protected:
90  void buffer_copy(string source, int pos)
91 {
92  for ( int i = 0; i < strlen(source); i++ )
93  header[pos+i] = source[i];
94 }
95 
96 public:
97 
98 
99 /**
100  * Create a header in the tarfile with name 'fname' and content 'content'.
101  *
102  * @param fname - the filename to store.
103  * @param content - the content of the file.
104  * @return the tar header for the filename.
105  */
106 protected:
107  string tar_header(string fname, string content)
108 {
109  int i, l;
110  string name, prefix;
111 
112  fname = replace
113  (fname,
114  ({ "ä", "ö", "ü", "Ä", "Ö", "Ü", "ß", "<", ">", "?", " ", "'" }),
115  ({ "\344", "\366", "\374", "\304", "\326", "\334", "\337",
116  "\74", "\76", "\77", "\40", "\47" }));
117 
118  if ( !stringp(content) )
119  l = 0;
120  else
121  l = strlen(content);
122 
123 
124  if ( strlen(fname) > 99 ) {
125  name = basename(fname);
126  prefix = dirname(fname);
127  }
128  else {
129  name = fname;
130  prefix = "\0";
131  }
132  if ( strlen(name) > 99 )
133  error("Cannot store names with more than 99 bytes: "+name+"\n");
134  if ( strlen(prefix) > 154 )
135  error("Cannot store files with prefix more than 154 chars.");
136 
137  header = "\0" * BLOCKSIZE;
138 
139  buffer_copy(name, 0);
140  buffer_copy("0100664", 100);
141  buffer_copy("0000767", 108);
142  buffer_copy("0000767", 116);
143  buffer_copy(to_oct(l, 12), 124);
144  buffer_copy(to_oct(time(), 12), 136);
145  int chksum = 7*32; // the checksum field is counted as ' '
146  buffer_copy("ustar ", 257);
147  buffer_copy("steam", 265);
148  buffer_copy("steam", 297);
149  buffer_copy(" 0", 155);
150 
151  buffer_copy(prefix, 345);
152 
153  for ( i = 0; i < BLOCKSIZE; i++ )
154  chksum += (0xff & header[i]);
155 
156  buffer_copy(to_oct(chksum, 7), 148);
157 
158  return header;
159 }
160 
161 public:
162 
163 /**
164  * Tar the content of the file 'fname'. Tars both header and content.
165  *
166  * @param string fname - the filename to tar.
167  * @param string content - the content of the file.
168  * @return the tared string.
169  */
170 string tar_content(string fname, string content)
171 {
172  string buf;
173  if ( !stringp(fname) || fname== "" ) {
174  FATAL("Empty file name !");
175  return "";
176  }
177  LOG("tar_content("+fname+", "+strlen(content)+" bytes)\n");
178  if ( !stringp(content) || strlen(content) == 0 )
179  return tar_header(fname, content);
180 
181  buf = tar_header(fname, content);
182  buf += content;
183  int rest = (strlen(content) % BLOCKSIZE);
184  if ( rest > 0 ) { // does not fit into a single buffer
185  rest = BLOCKSIZE - rest;
186  string b = "\0" * rest;
187  buf += b;
188  }
189  return buf;
190 }
191 
192 /**
193  * Create an end header for the tarfile.
194  *
195  * @return the end header.
196  */
197 string end_header()
198 {
199  return "\0" * BLOCKSIZE;
200 }
201 
202 /**
203  * Create an empty tarfile header.
204  *
205  * @return an empty tarfile header.
206  * @see tar_header
207  */
208 string empty_header()
209 {
210  return tar_header("", "");
211 }
212 
213 /**
214  * Create a tarfile with an array of given steam objects. This
215  * tars there identifiers and call the content functions.
216  *
217  * @param array arr - array of documents to be tared.
218  * @return the tarfile.
219  */
220 string tar_objects(array arr)
221 {
222  string tar = "";
223  foreach(arr, object obj) {
224  tar += tar_content(obj->get_identifier(), obj->get_content());
225  }
226  tar += end_header(); // empty header at the end
227  return tar;
228 }
229 
230 object open_file(string fname, mixed mode)
231 {
232  return ((program)"/kernel/DocFile")(OBJ(fname), mode);
233 }
234 
235 class steamfs {
236 public:
237  object open(string fname, string mode)
238  {
239  LOG_TAR("steamfs:open(%s)", fname);
240  object file = find_object(fname);
241  //return Stdio.FakeFile(file->get_content(), mode);
242  return open_file(fname, mode);
243  }
244  array get_dir(string dirname) {
245  array files = ({ });
246 
247  object cont = find_object(dirname);
248  array dir = cont->get_inventory();
249  foreach(dir, object o) {
250  if ( o->get_object_class() & CLASS_CONTAINER )
251  files += ({ o->get_identifier() + "/" });
252  else
253  files += ({ o->get_identifier() });
254  }
255  return files;
256  }
257  array get_files(string dirname) {
258  object cont = find_object(dirname);
259  return cont->get_inventory();
260  }
261 }
262 
263 protected:
264  array unpack_directory(string fs, string dirname, void|object contEnv)
265 {
266  array files;
267  array created = ({ });
268  object fsys = TarFile(fs);
269 
270  LOG_TAR("tar: unpack_directory(%s)", dirname);
271  files = fsys->get_dir(dirname);
272  foreach(files, string fname) {
273  //object file = TarFile(fs)->open(fname, "r");
274  object file = fsys->open(fname, "r");
275  if ( !objectp(file) )
276  steam_error("Failed to open: " + dirname + fname);
277 
278  array contfiles;
279  object cont, doc;
280 
281  if ( file->isdir() ) // container ?
282  {
283  cont = get_factory(CLASS_CONTAINER)->execute( ([ "name":
284  basename(fname), ]) );
285  contfiles = unpack_directory(fs, fname, cont);
286  foreach(contfiles, object dirfile)
287  dirfile->move(cont);
288  created += ({ cont });
289  }
290  else // ansonsten: File
291  {
292  fname = basename(fname);
293  LOG_TAR("tar: file(%s)", fname);
294  doc = get_factory(CLASS_DOCUMENT)->execute( (["name":fname, ]) );
295  if ( objectp(contEnv) )
296  doc->move(contEnv); // move before setting ocntent
297  doc->set_content( file->read() );
298  created += ({ doc });
299  LOG_TAR("tar: file(%s) has %d bytes", fname, doc->get_content_size());
300  }
301  file->close();
302 
303  }
304  return created;
305 }
306 
307 public:
308 
309 array unpack(string|object fname)
310 {
311  if ( objectp(fname) )
312  fname = get_module("filepath:tree")->object_to_filename(fname);
313  LOG_TAR("tar: unpack(%s)", fname);
314  array result = unpack_directory(fname, "");
315  return result;
316 }
317 
318 class _Tar // filesystem
319 {
320 public:
321  Stdio.File fd;
322  string filename;
323 
324  class ReadFile
325  {
326 
327  private int start, pos, len;
328  string _type;
329 
330 protected:
331  string _sprintf(int t)
332  {
333  return t=='O' && sprintf("Filesystem.Tar.ReadFile(%d, %d /* pos = %d */)",
334  start, len, pos);
335  }
336 
337 public:
338 
339  int seek(int p)
340  {
341  if(p<0)
342  if((p += len)<0)
343  p = 0;
344  if(p>=len) {
345  p = len-1;
346  if (!len) p = 0;
347  }
348  return ::seek((pos = p)+start);
349  }
350 
351  string read(int|void n)
352  {
353  if(!query_num_arg() || n>len-pos)
354  n = len-pos;
355  pos += n;
356  return ::read(n);
357  }
358 
359  void create(int p, int l, string type)
360  {
361  ::create(fd->get_document(), "r");
362  start = p;
363  len = l;
364  _type = type;
365  seek(0);
366  }
367  int isdir()
368  {
369  return !stringp(_type) || _type == "dir";
370  }
371  }
372 
373  class Record
374  {
375 
376  constant RECORDSIZE = 512;
377  constant NAMSIZ = 100;
378  constant TUNMLEN = 32;
379  constant TGNMLEN = 32;
380  constant SPARSE_EXT_HDR = 21;
381  constant SPARSE_IN_HDR = 4;
382 
383  string arch_name;
384 
385  int linkflag;
386  string arch_linkname;
387  string magic;
388  int devmajor;
389  int devminor;
390  int chksum;
391  string type;
392 
393  int pos;
394  int pseudo;
395 
396  // Header description:
397  //
398  // Fieldno Offset len Description
399  //
400  // 0 0 100 Filename
401  // 1 100 8 Mode (octal)
402  // 2 108 8 uid (octal)
403  // 3 116 8 gid (octal)
404  // 4 124 12 size (octal)
405  // 5 136 12 mtime (octal)
406  // 6 148 8 chksum (octal)
407  // 7 156 1 linkflag
408  // 8 157 100 linkname
409  // 9 257 8 magic
410  // 10 265 32 (USTAR) uname
411  // 11 297 32 (USTAR) gname
412  // 12 329 8 devmajor (octal)
413  // 13 337 8 devminor (octal)
414  // 14 345 167 (USTAR) Long path
415  //
416  // magic can be any of:
417  // "ustar\0""00" POSIX ustar (Version 0?).
418  // "ustar \0" GNU tar (POSIX draft)
419 
420  void create(void|string s, void|int _pos)
421  {
422  if(!s)
423  {
424  pseudo = 1;
425  return;
426  }
427 
428  pos = _pos;
429  array a = array_sscanf(s,
430  "%"+((string)NAMSIZ)+"s%8s%8s%8s%12s%12s%8s"
431  "%c%"+((string)NAMSIZ)+"s%8s"
432  "%"+((string)TUNMLEN)+"s"
433  "%"+((string)TGNMLEN)+"s%8s%8s%167s");
434  sscanf(a[0], "%s%*[\0]", arch_name);
435  sscanf(a[1], "%o", mode);
436  sscanf(a[2], "%o", uid);
437  sscanf(a[3], "%o", gid);
438  sscanf(a[4], "%o", size);
439  sscanf(a[5], "%o", mtime);
440  sscanf(a[6], "%o", chksum);
441  linkflag = a[7];
442  sscanf(a[8], "%s%*[\0]", arch_linkname);
443  sscanf(a[9], "%s%*[\0]", magic);
444 
445  if((magic=="ustar ") || (magic == "ustar"))
446  {
447  // GNU ustar or POSIX ustar
448  sscanf(a[10], "%s\0", uname);
449  sscanf(a[11], "%s\0", gname);
450  if (a[9] == "ustar\0""00") {
451  // POSIX ustar (Version 0?)
452  string long_path = "";
453  sscanf(a[14], "%s\0", long_path);
454  if (sizeof(long_path)) {
455  arch_name = long_path + "/" + arch_name;
456  }
457  } else if (arch_name == "././@LongLink") {
458  // GNU tar
459  // FIXME: Data contains full filename of next record.
460  }
461  }
462  else
463  uname = gname = 0;
464 
465  sscanf(a[12], "%o", devmajor);
466  sscanf(a[13], "%o", devminor);
467 
468  fullpath = combine_path_unix("/", arch_name);
469  name = (fullpath/"/")[-1];
470  atime = ctime = mtime;
471 
472  type =
473  ([ 0:"reg",
474  '0':"reg",
475  '1':0, // hard link
476  '2':"lnk",
477  '3':"chr",
478  '4':"blk",
479  '5':"dir",
480  '6':"fifo",
481  '7':0 // contigous
482  ])[linkflag] || "reg";
483  set_type(type);
484  }
485 
486  object open(string mode)
487  {
488  if(mode!="r")
489  error("Can only read right now.\n");
490  return ReadFile(pos, size, type);
491  }
492  };
493 
494  array(Record) entries = ({});
495  array filenames;
496  mapping filename_to_entry;
497 
498  void mkdirnode(string what, Record from, object parent)
499  {
500  Record r = Record();
501 
502  if(what=="") what = "/";
503 
504  r->fullpath = what;
505  r->name = (what/"/")[-1];
506 
507  r->mode = 0755|((from->mode&020)?020:0)|((from->mode&02)?02:0);
508  r->set_type("dir");
509  r->uid = 0;
510  r->gid = 0;
511  r->size = 0;
512  r->atime = r->ctime = r->mtime = from->mtime;
513  r->filesystem = parent;
514 
515  filename_to_entry[what] = r;
516  }
517 
518  void create(Stdio.File fd, string filename, object parent)
519  {
520  this_program::filename = filename;
521  // read all entries
522 
523  this_program::fd = fd;
524  int pos = 0; // fd is at position 0 here
525  for(;;)
526  {
527  Record r;
528  string s = this_program::fd->read(512);
529 
530  if(s=="" || strlen(s)<512 || sscanf(s, "%*[\0]%*2s")==1)
531  break;
532 
533  r = Record(s, pos+512);
534  r->filesystem = parent;
535 
536  if(r->arch_name!="") // valid file?
537  entries += ({ r });
538 
539  pos += 512 + r->size;
540  if(pos%512)
541  pos += 512 - (pos%512);
542  this_program::fd->seek(pos);
543  }
544 
545  filename_to_entry = mkmapping(entries->fullpath, entries);
546 
547  // create missing dirnodes
548 
549  array last = ({});
550  foreach(entries, Record r)
551  {
552  array path = r->fullpath/"/";
553  if(path[..sizeof(path)-2]==last) continue; // same dir
554  last = path[..sizeof(path)-2];
555 
556  for(int i = 0; i<sizeof(last); i++)
557  if(!filename_to_entry[last[..i]*"/"])
558  mkdirnode(last[..i]*"/", r, parent);
559  }
560 
561  filenames = indices(filename_to_entry);
562  }
563 
564  string _sprintf(int t)
565  {
566  return t=='O' && sprintf("_Tar(/* filename=%O */)", filename);
567  }
568 };
569 
570 class _TarFS
571 {
572 public:
573 
574  _Tar tar;
575 
576  Stdio.File fd; // tar file object
577  //not used; it's present in tar->filename, though /jhs 2001-01-20
578  // string filename; // tar filename in parent filesystem
579 
580  void create(void|_Tar _tar,
581  void|string _wd, void|string _root,
582  void|Filesystem.Base _parent)
583  {
584  tar = _tar;
585 
586  sscanf(reverse(_wd), "%*[\\/]%s", wd);
587  wd = reverse(wd);
588  if(wd=="")
589  wd = "/";
590 
591  sscanf(_root, "%*[/]%s", root);
592  parent = _parent;
593  }
594 
595  string _sprintf(int t)
596  {
597  return t=='O' && sprintf("_TarFS(/* root=%O, wd=%O */)", root, wd);
598  }
599 
600  Filesystem.Stat stat(string file, void|int lstat)
601  {
602  file = combine_path_unix(wd, file);
603  return tar->filename_to_entry[root+file];
604  }
605 
606  array get_dir(void|string directory, void|string|array globs)
607  {
608  directory = combine_path_unix(wd, (directory||""), "");
609 
610  array f = glob(root+directory+"?*", tar->filenames);
611  f -= glob(root+directory+"*/*", f); // stay here
612 
613  return f;
614  }
615 
616  Filesystem.Base cd(string directory)
617  {
618  Filesystem.Stat st = stat(directory);
619  if(!st) return 0;
620  if(st->isdir()) // stay in this filesystem
621  {
622  object new = _TarFS(tar, st->fullpath, root, parent);
623  return new;
624  }
625  return st->cd(); // try something else
626  }
627 
628  Stdio.File open(string filename, string mode)
629  {
630  LOG_TAR("fS:open(%s)",filename);
631  filename = combine_path_unix(wd, filename);
632  return tar->filename_to_entry[root+filename] &&
633  tar->filename_to_entry[root+filename]->open(mode);
634  }
635 
636  int access(string filename, string mode)
637  {
638  return 1; // sure
639  }
640 
641  int rm(string filename)
642  {
643  }
644 
645  void chmod(string filename, int|string mode)
646  {
647  }
648 
649  void chown(string filename, int|object owner, int|object group)
650  {
651  }
652 }
653 
654 class TarFile {
655 public:
656 
657  void create(string filename)
658  {
659  object parent = steamfs();
660  object fd = parent->open(filename, "r");
661  _Tar tar = _Tar(fd, filename, this_object());
662  _TarFS::create(tar, "/", "", parent);
663  }
664  string _sprintf(int t) {
665  return t=='O' && sprintf("TarFile(/* root=%O, wd=%O */)", root, wd);
666  }
667 }
668 
669 string get_identifier() { return "tar"; }
670 
671 
672 
673 
674 };