如何在具有1 TB磁盘空间的系统上从700 GB的txt文件中删除前3亿行？

#1 楼

gzip file && zcat file.gz | tail -n +300000001 | gzip > newFile.gz

#2 楼

$ seq 1 1000 > 1000lines.txt
$ head -n 3 1000lines.txt
1
2
3
$ tail -n 3 1000lines.txt
998
999
1000

$ stat -c %s 1000lines.txt
3893 # total bytes
$ head -n 300 1000lines.txt | wc -c
1092 # first 300 lines bytes
$ echo $((3893-1092))
2801 # target filesize after removal

$ dd conv=notrunc iflag=skip_bytes skip=1092 if=1000lines.txt of=1000lines.txt
5+1 records in
5+1 records out
2801 bytes (2.8 kB, 2.7 KiB) copied, 8.6078e-05 s, 32.5 MB/s

#3 楼

#4 楼

$ seq 1 1000 > 1000lines.txt
$ stat -c %s 1000lines.txt
3893 # total bytes
$ head -n 300 1000lines.txt | wc -c
1092 # first 300 lines bytes
$ echo $((3893-1092))
2801 # target filesize after removal

# losetup --find --show 1000lines.txt
/dev/loop0
losetup: 1000lines.txt: \
Warning: file does not fit into a 512-byte sector; \
the end of the file will be ignored.
# head -n 3 /dev/loop0
1 
2 
3 
# tail -n 3 /dev/loop0
921
922
923

# head -c 512 /dev/zero >> 1000lines.txt
# losetup --find --show 1000lines.txt
/dev/loop1
losetup: 1000lines.txt: \
Warning: file does not fit into a 512-byte sector; \
the end of the file will be ignored.
# tail -n 3 /dev/loop1
999
1000
# losetup --find --show --offset=1092 1000lines.txt
/dev/loop2
losetup: 1000lines.txt: \
Warning: file does not fit into a 512-byte sector; \
the end of the file will be ignored.
# head -n 3 /dev/loop2
301
302
303
# tail -n 3 /dev/loop2
999
1000
cp /dev/loop2 /dev/loop1

losetup -d /dev/loop2 /dev/loop1 /dev/loop0

truncate -s 2801 1000lines.txt

$ head -n 3 1000lines.txt 
301
302
303
$ tail -n 3 1000lines.txt 
998
999
1000

#5 楼

#6 楼

tail -n +300000001 < file 1<>; file

{
  tail -n +300000001 &&
    perl -e 'truncate STDOUT, tell STDOUT'
} < file 1<> file

#7 楼

/*  this file named    delete_n_lines.c

    compile by    gcc -W delete_n_lines.c -o delete_n_lines.x -lz

    have your huge text file already compressed via "gzip -9" to save disk space

    this program will also read a regular uncompressed text file
*/

# include <stdlib.h>
# include <stdio.h>
# include <string.h>
# include <zlib.h>

# define LL  2048   /* line length, number of characters up to '\n' */


int main ( int argc, char *argv[] )
{
   gzFile fin;
   FILE *fout;
   char line[LL];
   long int i, n = 0;
   long int n_lines_to_delete = 0;

   if ( argc != 4 )
   {
      printf("   Usage: %s  <infile> <outfile> <first_N_lines_to_delete>\n\n", argv[0] );
      exit( 0 );
   }

   n = sscanf( argv[3], "%d", &n_lines_to_delete );
   if ( n == 0 )
   {
      printf("\n   Error: problem reading N lines to delete\n\n" );
      exit( 0 );
   }

   if ( strcmp( argv[1], argv[2] ) == 0 )
   {
      printf("\n   Error: infile and outfile are the same.\n" );
      printf("          don't do that\n\n");
      exit( 0 );
   }

   fout = fopen( argv[2], "w" );
   if ( fout == NULL )
   {
      printf("\n   Error: could not write to %s\n\n", argv[2] );
      exit( 0 );
   }

   fin = gzopen( argv[1], "r" );
   if ( fin == NULL )
   {
      printf("\n   Error: could not read %s\n\n", argv[1] );
      fclose( fout );
      exit( 0 );
   }

   n = 0;
   gzgets( fin, line, LL );
   while ( ! gzeof( fin ) )
   {
      if ( n < n_lines_to_delete )
         n++;
      else
         fputs( line, fout );

      gzgets( fin, line, LL );
   }

   gzclose( fin );
   fclose( fout );

   printf("\n   deleted the first %d lines of %s, output file is %s\n\n", n, argv[1], argv[2] );


   return 0;
}

#8 楼

#9 楼

find the start of line 3 million and 1
create a new, empty file
repeat {
  read a decent number of blocks from the end of the old file
  append the blocks to the end of the new file
  truncate the old file by that many blocks
} until you get to the start of line 3 million and 1.

#10 楼

 function split_into_chunks {
    BIG_FILE=

    while [ $(stat -c %s $BIG_FILE) -gt 0 ]
    do
    CHUNK_FILE="chunk.$(ls chunk.* 2>/dev/null | wc -l)"
    tail -10 $BIG_FILE > $CHUNK_FILE
    test -s $CHUNK_FILE && truncate -s -$(stat -c %s $CHUNK_FILE) $BIG_FILE
    done
}

function concat_chunks {
    BIG_FILE=
    test ! -s $BIG_FILE || (echo "ERROR: target file is not empty"; return)

    for CHUNK_FILE in $(ls chunk.* | sort -t . -k2 -n -r)
    do
    cat $CHUNK_FILE >> $BIG_FILE
    rm $CHUNK_FILE
    done
}
 

$ seq 1000 > big-file.txt 
$ stat -c "%s %n" chunk.* big-file.txt 2>/dev/null | tail -12
3893 big-file.txt
$ md5sum big-file.txt; wc -l big-file.txt 
53d025127ae99ab79e8502aae2d9bea6  big-file.txt
1000 big-file.txt

$ split_into_chunks big-file.txt
$ stat -c "%s %n" chunk.* big-file.txt | tail -12
40 chunk.9
31 chunk.90
30 chunk.91
30 chunk.92
30 chunk.93
30 chunk.94
30 chunk.95
30 chunk.96
30 chunk.97
30 chunk.98
21 chunk.99
0 big-file.txt

$ # here you could change the chunks
$ # the test here shows that the file will be concatenated correctly again

$ concat_chunks big-file.txt
$ stat -c "%s %n" chunk.* big-file.txt 2>/dev/null | tail -12
3893 big-file.txt
$ md5sum big-file.txt; wc -l big-file.txt 
53d025127ae99ab79e8502aae2d9bea6  big-file.txt
1000 big-file.txt

#11 楼

#12 楼

RandomAccessFile out = new RandomAccessFile("file.txt", "rw");
RandomAccessFile in = new RandomAccessFile("file.txt", "r");
String line = null;
long rows = 0;
while( (line=in.readLine()) != null ){
    if( rows > 300000000 ) {
        out.writeBytes(line);
        out.write('\n');
    }
    rows++;
}
in.close();
out.setLength( out.getFilePointer() );
out.close();

#13 楼

 <?php
$fp1 = fopen("file.txt", "rb");
// find the position of the 3M'th line:
for ($i = 0; $i < 300_000_000; ++ $i) {
    fgets($fp1);
}
// the next fgets($fp1) call will read line 3M+1 :)
$fp2 = fopen("file.txt", "cb");
// copy all remaining lines from fp1 to fp2
while (false !== ($line = fgets($fp1))) {
    fwrite($fp2, $line);
}
fclose($fp1);
// remove every line that wasn't copied over to fp2
ftruncate($fp2, ftell($fp2));
fclose($fp2);
 

 #include <iostream>
#include <fstream>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>


int main(){
    const std::string target_file = "file.txt";
    std::fstream fp1(target_file, std::fstream::binary);
    fp1.exceptions(std::fstream::failbit | std::fstream::badbit);
    fp1.seekg(0, std::fstream::end);
    const std::streampos total_file_size_before_truncation = fp1.tellg();
    fp1.seekg(0, std::fstream::beg);
    const int fd = open(target_file.c_str(), O_RDWR);
    char *content_mmaped = (char *)mmap(NULL, total_file_size_before_truncation, PROT_READ, MAP_PRIVATE, fd, 0);
    const std::string_view content_view(content_mmaped, total_file_size_before_truncation);
    size_t line_no = 0;
    size_t line_pos = 0;
    size_t i = 0;
    for(; i < total_file_size_before_truncation; ++i){
        if(content_mmaped[i] == '\n'){
            ++line_no;
            line_pos = i;
            if(line_no >= (3000000-1)){
                break;
            }
        }
    }
    // idk why i have to do all those casts...
    fp1.write(&content_mmaped[i], std::streamoff(std::streamoff(total_file_size_before_truncation)-std::streamoff(i)));
    fp1.close();
    munmap(content_mmaped, total_file_size_before_truncation);
    ftruncate(fd, i);
    close(fd);
}