comments

README.md

@@ -23,24 +23,22 @@ There are two components needed to make this work:
 
 ```
 Usage of ./logveil:
+  -c value
+        Path to input file containing custom anonymization mappings
   -d value
         Path to directory with anonymizing data
+  -e    Change input file type to LM export (default: LM Backup)
   -i value
-        Path to input file containing logs to be anonymized (mandatory - if you don't specify input, code will fail)
+        Path to input file containing logs to be anonymized
   -o value
         Path to output file (default: Stdout)
-  -c value
-        Path to input file with custom anonymization mapping
-  -v
-        Enable verbose logging
-  -e
-        Change input file type to LM export (default: LM Backup)
-  -p
-        Enable proof writer (default: Disabled)
-  -r
-        Enable persistent (per session) replacement map (default: Disabled)
-  -h
-        Help for logveil
+  -p    Enable proof writer (default: Disabled)
+  -r    Enable persistent (per session) replacement map (default: Disabled)
+  -rs int
+        Size of the reader buffer in Bytes (default 4000000)
+  -v    Enable verbose logging (default: Disabled)
+  -ws int
+        Size of the writer buffer in Bytes (default 2000000)
 ```
 
 **Examples:**
@@ -164,10 +162,12 @@ And anonymization proof:
 {"original": "71:e5:41:18:cb:3e", "new": "0f:da:68:92:7f:2b"},
 ```
 
-## Replacement map and possible memory issues
+## Replacement map, possible memory issues and performance
 
 You can use `-r` flag to enable persistent replacement map. In such case LogVeil will keep replacement map in memory for each code run (per session) to make sure each unique value gets the same anonymized value each time it is encountered. Depending on the size of input data this replacement map can grow quite large which will cause degrading performance.
 
+In case of performance issues you can try using `rs` and `ws` flags to chage reader/writer buffer capacity - less I/O operations due to larger buffers should improve performance.
+
 ## Release
 
 Go to: https://github.com/logmanager-oss/logveil/releases to grab latest version of LogVeil. It is available for Windows, Linux and MacOS (x86_64/Arm64).

internal/config/config.go

@@ -16,6 +16,8 @@ type Config struct {
 	IsLmExport               bool
 	IsProofWriter            bool
 	IsPersistReplacementMap  bool
+	ReaderMaxCapacity        int
+	WriterMaxCapacity        int
 }
 
 // LoadAndValidate loads values from user supplied input into Config struct and validates them
@@ -33,6 +35,9 @@ func (c *Config) LoadAndValidate() {
 	flag.BoolVar(&c.IsProofWriter, "p", false, "Enable proof writer (default: Disabled)")
 	flag.BoolVar(&c.IsPersistReplacementMap, "r", false, "Enable persistent (per session) replacement map (default: Disabled)")
 
+	c.ReaderMaxCapacity = *flag.Int("rs", 4000000, "Size of the reader buffer in Bytes")
+	c.WriterMaxCapacity = *flag.Int("ws", 2000000, "Size of the writer buffer in Bytes")
+
 	flag.Parse()
 
 	// Check if mandatory flags are set

internal/proof/proof.go

@@ -30,8 +30,7 @@ func CreateProofWriter(config *config.Config, filesHandler *handlers.Files, buff
 		}
 		filesHandler.Add(file)
 
-		const maxCapacity = 1000000
-		proofWriter := bufio.NewWriterSize(file, maxCapacity)
+		proofWriter := bufio.NewWriterSize(file, config.WriterMaxCapacity)
 		buffersHandler.Add(proofWriter)
 
 		return &ProofWriter{

internal/reader/backup.go

@@ -24,15 +24,13 @@ type LmBackupReader struct {
 	file         *os.File
 }
 
-func NewLmBackupReader(input *os.File) (*LmBackupReader, error) {
+func NewLmBackupReader(input *os.File, maxCapacity int) (*LmBackupReader, error) {
 	gzReader, err := gzip.NewReader(input)
 	if err != nil {
 		return nil, fmt.Errorf("error creating gzip reader: %w", err)
 	}
 
 	backupReader := bufio.NewScanner(gzReader)
-	//adjust the capacity to your need (max characters in line)
-	const maxCapacity = 4000000
 	buf := make([]byte, maxCapacity)
 	backupReader.Buffer(buf, maxCapacity)
 

internal/reader/backup_test.go

@@ -39,7 +39,8 @@ func TestLmBackup(t *testing.T) {
 			}
 			defer inputFile.Close()
 
-			inputReader, err := NewLmBackupReader(inputFile)
+			maxCapacity := 4000000
+			inputReader, err := NewLmBackupReader(inputFile, maxCapacity)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/reader/reader.go

@@ -28,7 +28,7 @@ func CreateInputReader(config *config.Config, filesHandler *handlers.Files) (Inp
 		return inputReader, nil
 	}
 
-	inputReader, err := NewLmBackupReader(inputFile)
+	inputReader, err := NewLmBackupReader(inputFile, config.ReaderMaxCapacity)
 	if err != nil {
 		return nil, fmt.Errorf("initializin LM Backup reader: %v", err)
 	}

internal/writer/writer.go

@@ -25,8 +25,7 @@ func CreateOutputWriter(config *config.Config, filesHandler *handlers.Files, buf
 		outputFile = os.Stdout
 	}
 
-	const maxCapacity = 2000000
-	outputWriter := bufio.NewWriterSize(outputFile, maxCapacity)
+	outputWriter := bufio.NewWriterSize(outputFile, config.WriterMaxCapacity)
 
 	buffersHandler.Add(outputWriter)