Commit b39edb62 authored by Oleh Astappiev's avatar Oleh Astappiev
Browse files

Flush data to database less frequently

parent bb980a0b
......@@ -31,6 +31,7 @@ public class Runner {
}
private static int processInput(File inputFile){
long processedUrls = 0;
try {
Scanner scanner = new Scanner(inputFile);
logger.warn("BRKN Reading input file from start...");
......@@ -44,12 +45,16 @@ public class Runner {
logger.error("Can't process url: {}", url);
throw e;
} finally {
databaseLogger.flush();
processedUrls++;
if (processedUrls % 100 == 0) {
databaseLogger.flush();
}
}
}
return 0;
} catch (Throwable e) {
logger.error("An error occurred: ", e);
logger.error("An error occurred on line {}: ", processedUrls + 1, e);
return 1;
} finally {
databaseLogger.clean();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment