Performance tuning - backend

This commit is contained in:
2026-07-21 21:32:53 +05:30
parent 4f7c06ed97
commit f0481d090c
14 changed files with 330 additions and 139 deletions

View File

@@ -5,6 +5,10 @@
<display-name>Matrix Cygnus 1.0</display-name>
<listener>
<listener-class>matrix.nimble.utilities.DatabasePoolLifecycleListener</listener-class>
</listener>
<!-- The front controller of the Spring MVC Web application, responsible
for handling all application requests -->
<servlet>

View File

@@ -21,6 +21,7 @@
<poi.version>5.5.1</poi.version>
<itext.version>5.3.4</itext.version>
<log4j.version>2.24.3</log4j.version>
<hikaricp.version>6.3.3</hikaricp.version>
<junit.version>5.12.2</junit.version>
<tomcat.version>10.1.57</tomcat.version>
<server.port>8080</server.port>
@@ -123,6 +124,11 @@
<version>42.7.13</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>${hikaricp.version}</version>
</dependency>
<dependency>
<groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId>

View File

@@ -324,7 +324,7 @@ public class CombinedExcelReporting {
}
if(isProcessFlag())
{
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail sent successfully.\n" : ""));
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail delivery queued.\n" : ""));
}
}
private void DesignReport(String [] reportData,String [][] reportlayout)
@@ -430,15 +430,11 @@ public class CombinedExcelReporting {
{
if(isAutoemail)
{
SendReports SR=new SendReports();
SR.setErrCode(getErrCode());
SR.setProcessFlag(true);
SR.SendReportsByBatch(processedOn, BranchId,getUserId(),runningMode);
if (!SR.isProcessFlag())
if (!ReportEmailJobExecutor.submitBatch(getErrCode(), processedOn, BranchId, getUserId(), runningMode))
{
setErrMsg(SR.getErrMsg().replace(getErrCode()+"SEML:warning:", getErrCode()+"SEML:warning:Bank Reports generated successfully.\n\n"));
setErrDesc(SR.getErrDesc());
setProcessFlag(SR.isProcessFlag());
setErrMsg(getErrCode()+"SEML:warning:Bank Reports generated successfully, but the email queue is full. Please retry from the mailbox.");
setErrDesc("The bounded report email queue has reached its configured capacity.");
setProcessFlag(false);
}
}
}

View File

@@ -314,7 +314,7 @@ public class CombinedReporting {
}
if(isProcessFlag())
{
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail sent successfully.\n" : ""));
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail delivery queued.\n" : ""));
}
}
private void DesignReport(String [] reportData,String [][] reportlayout,PDFFormatter PDFF)
@@ -482,15 +482,11 @@ public class CombinedReporting {
{
if(isAutoemail)
{
SendReports SR=new SendReports();
SR.setErrCode(getErrCode());
SR.setProcessFlag(true);
SR.SendReportsByBatch(processedOn, BranchId,getUserId(),runningMode);
if (!SR.isProcessFlag())
if (!ReportEmailJobExecutor.submitBatch(getErrCode(), processedOn, BranchId, getUserId(), runningMode))
{
setErrMsg(SR.getErrMsg().replace(getErrCode()+"SEML:warning:", getErrCode()+"SEML:warning:Bank Reports generated successfully.\n\n"));
setErrDesc(SR.getErrDesc());
setProcessFlag(SR.isProcessFlag());
setErrMsg(getErrCode()+"SEML:warning:Bank Reports generated successfully, but the email queue is full. Please retry from the mailbox.");
setErrDesc("The bounded report email queue has reached its configured capacity.");
setProcessFlag(false);
}
}
}

View File

@@ -287,7 +287,7 @@ public class CombinedWordlReporting {
}
if(isProcessFlag())
{
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail sent successfully.\n" : ""));
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail delivery queued.\n" : ""));
}
}
private void DesignReport(String [] reportData,String [][] reportlayout)
@@ -488,15 +488,11 @@ public class CombinedWordlReporting {
{
if(isAutoemail)
{
SendReports SR=new SendReports();
SR.setErrCode(getErrCode());
SR.setProcessFlag(true);
SR.SendReportsByBatch(processedOn, BranchId,getUserId(),runningMode);
if (!SR.isProcessFlag())
if (!ReportEmailJobExecutor.submitBatch(getErrCode(), processedOn, BranchId, getUserId(), runningMode))
{
setErrMsg(SR.getErrMsg().replace(getErrCode()+"SEML:warning:", getErrCode()+"SEML:warning:Bank Reports generated successfully.\n\n"));
setErrDesc(SR.getErrDesc());
setProcessFlag(SR.isProcessFlag());
setErrMsg(getErrCode()+"SEML:warning:Bank Reports generated successfully, but the email queue is full. Please retry from the mailbox.");
setErrDesc("The bounded report email queue has reached its configured capacity.");
setProcessFlag(false);
}
}
}

View File

@@ -287,7 +287,7 @@ public class CombinedWordlReportingTabular {
}
if(isProcessFlag())
{
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail sent successfully.\n" : ""));
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail delivery queued.\n" : ""));
}
}
private void DesignReport(String [] reportData,String [][] reportlayout)
@@ -488,15 +488,11 @@ public class CombinedWordlReportingTabular {
{
if(isAutoemail)
{
SendReports SR=new SendReports();
SR.setErrCode(getErrCode());
SR.setProcessFlag(true);
SR.SendReportsByBatch(processedOn, BranchId,getUserId(),runningMode);
if (!SR.isProcessFlag())
if (!ReportEmailJobExecutor.submitBatch(getErrCode(), processedOn, BranchId, getUserId(), runningMode))
{
setErrMsg(SR.getErrMsg().replace(getErrCode()+"SEML:warning:", getErrCode()+"SEML:warning:Bank Reports generated successfully.\n\n"));
setErrDesc(SR.getErrDesc());
setProcessFlag(SR.isProcessFlag());
setErrMsg(getErrCode()+"SEML:warning:Bank Reports generated successfully, but the email queue is full. Please retry from the mailbox.");
setErrDesc("The bounded report email queue has reached its configured capacity.");
setProcessFlag(false);
}
}
}

View File

@@ -155,7 +155,7 @@ public class PartWiseReporting {
}
if(isProcessFlag())
{
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail sent successfully.\n" : ""));
setErrMsg(getErrCode()+"REPGEN:Info:Reports generated successfully."+(RS.isAutoemail() ? "\n\nEmail delivery queued.\n" : ""));
}
}
private boolean CreateReport(String [][] reportData,String [][] reportlayout,String reportType,String rootPath,String curDate,String processedOn,String reportBy,String repType,String UserId,String BranchId,boolean isArchive,String runningMode,String portfolioid)
@@ -405,15 +405,11 @@ public class PartWiseReporting {
{
if(isAutoemail)
{
SendReports SR=new SendReports();
SR.setErrCode(getErrCode());
SR.setProcessFlag(true);
SR.SendReportsByBatch(processedOn, BranchId, UserId,runningMode);
if (!SR.isProcessFlag())
if (!ReportEmailJobExecutor.submitBatch(getErrCode(), processedOn, BranchId, UserId, runningMode))
{
setErrMsg(SR.getErrMsg().replace(getErrCode()+"SEML:warning:", getErrCode()+"SEML:warning:Bank Reports generated successfully.\n\n"));
setErrDesc(SR.getErrDesc());
setProcessFlag(SR.isProcessFlag());
setErrMsg(getErrCode()+"SEML:warning:Bank Reports generated successfully, but the email queue is full. Please retry from the mailbox.");
setErrDesc("The bounded report email queue has reached its configured capacity.");
setProcessFlag(false);
}
}
}

View File

@@ -0,0 +1,82 @@
package matrix.nimble.operation.output.model;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/** Bounded background executor for automatically generated report-email batches. */
public final class ReportEmailJobExecutor {
private static final int WORKERS = integerSetting("matrix.mail.workers", 2);
private static final int QUEUE_CAPACITY = integerSetting("matrix.mail.queueCapacity", 50);
private static final AtomicInteger THREAD_SEQUENCE = new AtomicInteger();
private static final ThreadPoolExecutor EXECUTOR = new ThreadPoolExecutor(
WORKERS,
WORKERS,
0L,
TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(QUEUE_CAPACITY),
new ReportEmailThreadFactory(),
new ThreadPoolExecutor.AbortPolicy());
private ReportEmailJobExecutor() {
}
public static boolean submitBatch(String errorCode, String batch, String branchId,
String userId, String runningMode) {
try {
EXECUTOR.execute(() -> sendBatch(errorCode, batch, branchId, userId, runningMode));
return true;
} catch (RejectedExecutionException rejected) {
System.err.println("Report email queue is full; batch was not queued: " + batch);
return false;
}
}
private static void sendBatch(String errorCode, String batch, String branchId,
String userId, String runningMode) {
try {
SendReports reports = new SendReports();
reports.setErrCode(errorCode);
reports.setProcessFlag(true);
reports.SendReportsByBatch(batch, branchId, userId, runningMode);
if (!reports.isProcessFlag()) {
System.err.println("Report email batch completed with failures: " + reports.getErrMsg());
}
} catch (RuntimeException exception) {
System.err.println("Report email batch failed unexpectedly: " + exception.getMessage());
exception.printStackTrace(System.err);
}
}
public static void shutdown() {
EXECUTOR.shutdown();
try {
if (!EXECUTOR.awaitTermination(10, TimeUnit.SECONDS)) {
EXECUTOR.shutdownNow();
}
} catch (InterruptedException interrupted) {
EXECUTOR.shutdownNow();
Thread.currentThread().interrupt();
}
}
private static int integerSetting(String name, int defaultValue) {
int value = Integer.parseInt(System.getProperty(name, Integer.toString(defaultValue)));
if (value < 1) {
throw new IllegalArgumentException(name + " must be greater than zero");
}
return value;
}
private static final class ReportEmailThreadFactory implements ThreadFactory {
@Override
public Thread newThread(Runnable task) {
Thread thread = new Thread(task, "cygnus-report-email-" + THREAD_SEQUENCE.incrementAndGet());
thread.setDaemon(true);
return thread;
}
}
}

View File

@@ -137,9 +137,7 @@ public class DBFunctions {
}
private void ConnectDB() {
try {
Class.forName(ModFunc.decrypt(FFunc.getDBDriver())).newInstance();
//con = DriverManager.getConnection(ModFunc.decrypt(FFunc.getConnString()), ModFunc.decrypt(FFunc.getDBUser()), ModFunc.decrypt(FFunc.getDBPassword()));
con = DriverManager.getConnection("jdbc:postgresql://192.168.0.111:5432/matrix", "postgres", "M@triXPostgr3s@6202");
con = DatabaseConnectionPool.getConnection();
} catch (Exception exce) {
setErrMsg(getErrCode()+"CONDB:Error:Could not connect with database. Please check the network connection.");
setErrDesc(exce.getMessage());
@@ -474,16 +472,16 @@ public class DBFunctions {
String [][] dataArray=null;
Hashtable<String , String> dataHash=new Hashtable<String, String>();
int ColCount=0;
String Result="";
StringBuilder result = new StringBuilder();
int RowIndex=0;
String tempStr="";
String hashKey="";
String hashValue="";
StringBuilder hashValue = new StringBuilder();
int hascol=getHashKeyColIndex();
boolean ishash=isGenerateHash();
while (RstSet.next())
{
hashValue="";
hashValue.setLength(0);
if(RowIndex==0)
{
rsmd = RstSet.getMetaData();
@@ -499,21 +497,21 @@ public class DBFunctions {
{
hashKey=tempStr;
}
hashValue+=tempStr+ColDelimeter;
hashValue.append(tempStr).append(ColDelimeter);
}
Result+=tempStr+ColDelimeter;
result.append(tempStr).append(ColDelimeter);
dataArray[RowIndex][i-1]=tempStr;
}
if(ishash)
{
dataHash.put(hashKey, hashValue);
dataHash.put(hashKey, hashValue.toString());
}
Result+=RowDelimeter;
result.append(RowDelimeter);
RowIndex++;
}
setResultHash(dataHash);
setResultArray(dataArray);
return Result;
return result.toString();
}
private static int getRowCount(ResultSet set) throws SQLException
{
@@ -535,10 +533,9 @@ public class DBFunctions {
ConnectDB();
if(isProcessFlag())
{
try
try (CallableStatement proc = con.prepareCall("{ ? = call "+ProcedureName+" }"))
{
con.setAutoCommit(false);
CallableStatement proc =con.prepareCall("{ ? = call "+ProcedureName+" }");
proc.registerOutParameter(1, Types.LONGVARCHAR);
if(ParameterList.isEmpty()){}
else
@@ -555,11 +552,24 @@ public class DBFunctions {
}
catch(Exception excp)
{
try {
if (con != null) {
con.rollback();
}
} catch (SQLException rollbackException) {
setErrDesc(excp.getMessage()+"; rollback failed: "+rollbackException.getMessage());
}
setProcessFlag(false);
setErrMsg(getErrCode()+"CLPROC:Error:Record updation failed. Please try again later");
setErrDesc(excp.getMessage());
if (getErrDesc() == null || !getErrDesc().contains("rollback failed")) {
setErrDesc(excp.getMessage());
}
Result=getErrMsg();
}
finally
{
CloseConnection();
}
}
}
return Result;

View File

@@ -0,0 +1,70 @@
package matrix.nimble.utilities;
import java.sql.Connection;
import java.sql.SQLException;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
/** Shared, bounded JDBC connection pool for the legacy database layer. */
public final class DatabaseConnectionPool {
private static final String DEFAULT_JDBC_URL = "jdbc:postgresql://192.168.0.111:5432/matrix";
private static final String DEFAULT_USERNAME = "postgres";
private static final String DEFAULT_PASSWORD = "M@triXPostgr3s@6202";
private static volatile HikariDataSource dataSource;
private DatabaseConnectionPool() {
}
public static Connection getConnection() throws SQLException {
return getDataSource().getConnection();
}
private static HikariDataSource getDataSource() {
HikariDataSource current = dataSource;
if (current != null) {
return current;
}
synchronized (DatabaseConnectionPool.class) {
if (dataSource == null) {
HikariConfig config = new HikariConfig();
config.setPoolName("CygnusDatabasePool");
config.setJdbcUrl(setting("matrix.db.url", "MATRIX_DB_URL", DEFAULT_JDBC_URL));
config.setUsername(setting("matrix.db.username", "MATRIX_DB_USERNAME", DEFAULT_USERNAME));
config.setPassword(setting("matrix.db.password", "MATRIX_DB_PASSWORD", DEFAULT_PASSWORD));
config.setMaximumPoolSize(integerSetting("matrix.db.maximumPoolSize", 10));
config.setMinimumIdle(integerSetting("matrix.db.minimumIdle", 2));
config.setConnectionTimeout(longSetting("matrix.db.connectionTimeoutMs", 30_000L));
config.setValidationTimeout(longSetting("matrix.db.validationTimeoutMs", 5_000L));
config.setIdleTimeout(longSetting("matrix.db.idleTimeoutMs", 600_000L));
config.setMaxLifetime(longSetting("matrix.db.maxLifetimeMs", 1_800_000L));
config.setLeakDetectionThreshold(longSetting("matrix.db.leakDetectionThresholdMs", 60_000L));
dataSource = new HikariDataSource(config);
}
return dataSource;
}
}
public static synchronized void close() {
if (dataSource != null) {
dataSource.close();
dataSource = null;
}
}
private static String setting(String systemProperty, String environmentVariable, String defaultValue) {
String value = System.getProperty(systemProperty);
if (value == null || value.isBlank()) {
value = System.getenv(environmentVariable);
}
return value == null || value.isBlank() ? defaultValue : value;
}
private static int integerSetting(String name, int defaultValue) {
return Integer.parseInt(System.getProperty(name, Integer.toString(defaultValue)));
}
private static long longSetting(String name, long defaultValue) {
return Long.parseLong(System.getProperty(name, Long.toString(defaultValue)));
}
}

View File

@@ -0,0 +1,14 @@
package matrix.nimble.utilities;
import jakarta.servlet.ServletContextEvent;
import jakarta.servlet.ServletContextListener;
import matrix.nimble.operation.output.model.ReportEmailJobExecutor;
/** Releases pooled database resources when Tomcat stops or reloads the application. */
public class DatabasePoolLifecycleListener implements ServletContextListener {
@Override
public void contextDestroyed(ServletContextEvent event) {
ReportEmailJobExecutor.shutdown();
DatabaseConnectionPool.close();
}
}

View File

@@ -4,13 +4,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.channels.SocketChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.Random;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -29,8 +25,6 @@ import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
import com.fasterxml.jackson.databind.ObjectMapper;
public class EmailFunctions {
private String username;
private String password;
@@ -209,6 +203,9 @@ public class EmailFunctions {
Properties props=new Properties();
props.setProperty("mail.smtp.auth", "true");
props.setProperty("mail.smtp.starttls.enable", "true");
props.setProperty("mail.smtp.connectiontimeout", System.getProperty("matrix.mail.connectionTimeoutMs", "30000"));
props.setProperty("mail.smtp.timeout", System.getProperty("matrix.mail.readTimeoutMs", "30000"));
props.setProperty("mail.smtp.writetimeout", System.getProperty("matrix.mail.writeTimeoutMs", "30000"));
if(getUsername().toLowerCase().contains("@arveebizpro.com")) {
@@ -272,11 +269,8 @@ public class EmailFunctions {
}
}
);
ObjectMapper Obj = new ObjectMapper();
String sessionStr = "";
try
{
sessionStr = Obj.writeValueAsString(session);
Message message = new MimeMessage(session);
message.setFrom(new InternetAddress(getFrom()));
message.setRecipients(Message.RecipientType.TO,InternetAddress.parse(getTo()));
@@ -309,25 +303,18 @@ public class EmailFunctions {
}
// Send the complete message parts
message.setContent(multipart);
if(CheckInternetConnection("www.google.co.in"))
{
Transport.send(message);
}
else
{
return "Error - Unable to access internet. Please check your internet connection";
}
Transport.send(message);
/*if(isArchive())
{
new File(FileList.get(0)).delete();
}*/
}catch(IOException iexce)
{
return "Error - Request Sent to : "+sessionStr + ", Response IO: " + iexce.getMessage();
return "Error - Unable to prepare email attachments: " + iexce.getMessage();
}
catch(MessagingException exce)
{
return "Error - Request Sent to : "+sessionStr + ", Response Message: " + exce.getMessage();
return "Error - Unable to send email: " + exce.getMessage();
}
}
else
@@ -402,19 +389,4 @@ public class EmailFunctions {
FileList=new ArrayList<String>();
FileList.add(zipFile);
}
public boolean CheckInternetConnection(String UrlToCheck)
{
try
{
SocketChannel channel;
InetSocketAddress socketAddress = new InetSocketAddress(UrlToCheck, 80);
channel = SocketChannel.open();
channel.configureBlocking(false);
channel.connect(socketAddress);
return true;
}catch(Exception exce)
{
return false;
}
}
}

View File

@@ -4,8 +4,15 @@ import java.io.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
public class FileFunctions {
private static final Object QUERY_CACHE_LOCK = new Object();
private static volatile Map<Integer, String> queryCache = Collections.emptyMap();
private static volatile long queryCacheLastModified = Long.MIN_VALUE;
private static volatile String queryCachePath = "";
private String ConnString;
private String DBDriver;
private String DBUser;
@@ -212,15 +219,41 @@ public class FileFunctions {
{
realPath=realPath.replace("%20", " ");
}
File QueryFile = new File(realPath + "nimble.qry");
BufferedReader Breader = new BufferedReader(new FileReader(QueryFile));
String TempContent;
while ((TempContent = Breader.readLine()) != null) {
if (TempContent.split(GlobalClass.ColDelim)[0].equals("Query"+QueryIndex)) {
return TempContent.replace("Query"+QueryIndex+GlobalClass.ColDelim, "");
}
File queryFile = new File(realPath + "nimble.qry");
refreshQueryCacheIfRequired(queryFile);
return queryCache.get(QueryIndex);
}
private static void refreshQueryCacheIfRequired(File queryFile) throws IOException {
String absolutePath = queryFile.getAbsolutePath();
long lastModified = queryFile.lastModified();
if (absolutePath.equals(queryCachePath) && lastModified == queryCacheLastModified) {
return;
}
synchronized (QUERY_CACHE_LOCK) {
if (absolutePath.equals(queryCachePath) && lastModified == queryCacheLastModified) {
return;
}
Map<Integer, String> loadedQueries = new HashMap<>();
try (BufferedReader reader = new BufferedReader(new FileReader(queryFile))) {
String line;
while ((line = reader.readLine()) != null) {
int delimiterIndex = line.indexOf(GlobalClass.ColDelim);
if (delimiterIndex <= 5 || !line.startsWith("Query")) {
continue;
}
try {
int queryIndex = Integer.parseInt(line.substring(5, delimiterIndex));
loadedQueries.put(queryIndex, line.substring(delimiterIndex + GlobalClass.ColDelim.length()));
} catch (NumberFormatException ignored) {
// Ignore malformed/non-query lines, matching the legacy lookup behavior.
}
}
}
queryCache = Collections.unmodifiableMap(loadedQueries);
queryCachePath = absolutePath;
queryCacheLastModified = lastModified;
}
Breader.close();
return TempContent;
}
}

View File

@@ -0,0 +1,20 @@
package matrix.nimble;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
import matrix.nimble.utilities.FileFunctions;
class FileFunctionsQueryCacheTest {
@Test
void loadsQueriesFromNimbleQueryFileAndHandlesMissingCodes() throws Exception {
FileFunctions files = new FileFunctions("TEST");
String query = files.GetQuery(1);
assertTrue(query.startsWith("select!C0L!select u.user_id"));
assertNull(files.GetQuery(Integer.MAX_VALUE));
}
}