Sunday, July 22, 2018

[Bash] Split large files into smaller files


division.sh
================================================================
#!/bin/bash

mFileIn="big_log.txt";
#mFileIn=$1;
mDate=`date '+%Y%m%d%H%M'`;
mCount=1000;
mTotal=`cat -v $mFileIn | wc -l`;
mLoop=$((mTotal / mCount + 1));
echo $mFileIn;
echo $mDate;
echo $mCount;
echo $mTotal;
echo $mLoop;

for (( i=0; i < mLoop; i++ ))
do
        mFileOut=`printf "small_log_""$mDate""_%0.5i.txt" $i`;
        mStart=$((mCount * i + 1));
        mEnd=$((mStart + mCount - 1));
        #echo $mFileOut;
        echo $mStart " ~ " $mEnd;
        echo $mCmd;
        eval $mCmd;

        mCmd=`printf "sed -n -e '$mStart,$mEnd p' $mFileIn > $mFileOut"`;
        echo $mCmd;
        eval $mCmd;
done
================================================================

OR just

$ split -l 1000000 big_log.txt