Resolve out of memory error

I have made a java program that connects to a Progress SmartDataObject using an AppServer. The SmartDataObject gives me a resultset from a (Progress) database. Currently I am testing with a table with 1 million entries.

I have to copy the recs from Progress to a Postgres database and update the progress database (set some status from 10 to 80).

Testing with an amount of 40000 records worked fine.

But I keep on getting problems with the heap size with this current test. I have tried -Xms2g, 4g, -Xmx4g, after that left it out.

I have no control over the amount of records I get from the Progress database. I use batch updates towards both databases.

(I am not a highly experienced java programmer)


` public static void main(String[] args) {
final String WRITEREC = "INSERT INTO test "
+ "(type, tabel, opvoerusr, opvoerts, huidigjson, rowid, vorigjson, uniekeindex, verstuurdts) "
+ "VALUES (?,?,?,?,?,?,?,?,?) LIMIT 10000"
;



boolean lOke = true;
long start = System.currentTimeMillis();

Properties properties = new Properties();
URL url = ClassLoader.getSystemResource("config.properties");
try {
properties.load(url.openStream());
} catch (IOException e) {
System.out.println("Fout bij openen config.properties: " + e.getMessage());
lOke = false;
return;
}

try {
Class.forName("org.postgresql.Driver").newInstance();
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
System.out.println("Fout bij initiëren verbinding met Postgres: " + e.getMessage());
lOke = false;
return;
}

String SDOApp;
String myOS = System.getProperty("os.name");

if (myOS.contains("Windows")) {
SDOApp = properties.getProperty("SDOApp_Windows");
}
else {
SDOApp = properties.getProperty("SDOApp_Linux");
}

String SDO = properties.getProperty("SmartDataObject");

SDOAppObject appObj;
SDOResultSet rs = null;

PreparedStatement statement;
Timestamp ts = new Timestamp(System.currentTimeMillis());
Connection con = null;

try {
appObj = new SDOAppObject(SDOApp, "", "", "");
rs = appObj._createSDOResultSet(SDO);
appObj._release();
} catch (Open4GLException | ProSQLException | IOException e) {
System.out.println("Fout bij ophalen gegevens uit Progress: " + e.getMessage());
lOke = false;
return;
}

try {
con = DriverManager.getConnection(properties.getProperty("Host"), properties.getProperty("User"), properties.getProperty("Wachtwoord"));
statement = con.prepareStatement(WRITEREC, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE);
con.setAutoCommit(false);
} catch (SQLException e) {
System.out.println("Fout bij connecten AppServer: " + e.getMessage());
lOke = false;
return;
}

PGobject jsonObject = new PGobject();
jsonObject.setType("jsonb");

try {
while (rs.next()) {
/* Postgres */
statement.setString(1, rs.getString("cType"));
statement.setString(2, rs.getString("cTabel"));
statement.setString(3, rs.getString("cOpgevoerd"));
statement.setTimestamp(4, rs.getTimestamp("dtOpgevoerd"));
statement.setString(6, rs.getString("cRowid"));
statement.setString(8, rs.getString("cPrimaryUniqueIndex"));
statement.setTimestamp(9, ts);

jsonObject.setValue(rs.getString("cHuidig"));
statement.setObject(5, jsonObject);
// if ((jsonString != null) & (!jsonString.isEmpty())) {
// jsonObject.setValue(jsonString);
// }

jsonObject.setValue(rs.getString("cVorig"));
statement.setObject(7, jsonObject);

// if ((jsonString != null) & (!jsonString.isEmpty())) {
// jsonObject.setValue(rs.getString("cVorig"));
// }

statement.addBatch();

/* Progress */
rs.updateInt("iStatus", 80);
rs.updateRow();
}

rs.sendBatch();
statement.executeBatch();

} catch (ProSQLException e) {
System.out.println("Fout bij updaten Progress database: " + e.getMessage());
lOke = false;
} catch (OutOfMemoryError e) {
System.out.println("Geheugenfout: " + e.getMessage());
lOke = false;
} catch (SQLException e) {
System.out.println("Fout bij samenstellen gegevens voor Postgres: " + e.getMessage());
lOke = false;
}

finally {
long millis = System.currentTimeMillis() - start;
long second = (millis / 1000) % 60;
long minute = (millis / (1000 * 60)) % 60;
long hour = (millis / (1000 * 60 * 60)) % 24;
String time = format("%02d:%02d:%02d:%d", hour, minute, second, millis);

if (lOke) {
System.out.println("Verwerking afgerond " + time);

} else {
try {
if (!con.isClosed()) {
con.rollback();
}
} catch (SQLException e) {
System.out.println("Fout bij terugdraaien batch naar Postgres " + e.getMessage());
}

System.out.println("Programma afgerond. Verwerking is mogelijk incompleet of niet uitgevoerd " + time);
}

try {
rs.close();
statement.close();

con.setAutoCommit(true);
} catch (SQLException e) {}
}`

Continue reading...