Sunday, July 22, 2018
[Bash] Split large files into smaller files
division.sh
================================================================
#!/bin/bash
mFileIn="big_log.txt";
#mFileIn=$1;
mDate=`date '+%Y%m%d%H%M'`;
mCount=1000;
mTotal=`cat -v $mFileIn | wc -l`;
mLoop=$((mTotal / mCount + 1));
echo $mFileIn;
echo $mDate;
echo $mCount;
echo $mTotal;
echo $mLoop;
for (( i=0; i < mLoop; i++ ))
do
mFileOut=`printf "small_log_""$mDate""_%0.5i.txt" $i`;
mStart=$((mCount * i + 1));
mEnd=$((mStart + mCount - 1));
#echo $mFileOut;
echo $mStart " ~ " $mEnd;
echo $mCmd;
eval $mCmd;
mCmd=`printf "sed -n -e '$mStart,$mEnd p' $mFileIn > $mFileOut"`;
echo $mCmd;
eval $mCmd;
done
================================================================
OR just
$ split -l 1000000 big_log.txt