From eea002baffd9aec2ff5f35d0297bb0a33b8f45d3 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 14 Jul 2026 16:09:44 +0530 Subject: [PATCH 01/81] fix: add clinical staff information --- .../data/nurse/BeneficiaryVisitDetail.java | 48 +++++++++++++++++++ .../iemr/mmu/repo/login/UserLoginRepo.java | 3 ++ .../mmu/repo/nurse/BenVisitDetailRepo.java | 12 +++++ .../transaction/CommonDoctorServiceImpl.java | 34 +++++++++++++ .../transaction/CommonNurseServiceImpl.java | 20 ++++++++ .../LabTechnicianServiceImpl.java | 22 +++++++++ 6 files changed, 139 insertions(+) diff --git a/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java b/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java index dd17ef7f..ff7778ec 100644 --- a/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java +++ b/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java @@ -141,6 +141,22 @@ public class BeneficiaryVisitDetail { @Column(name = "VisitFlowStatusFlag", insertable = false) private String visitFlowStatusFlag; + @Expose + @Column(name = "NurseID") + private Long nurseID; + + @Expose + @Column(name = "DoctorID") + private Long doctorID; + + @Expose + @Column(name = "PharmacistID") + private Long pharmacistID; + + @Expose + @Column(name = "LabTechnicianID") + private Long labTechnicianID; + @Expose @Column(name = "VanSerialNo") private Long vanSerialNo; @@ -535,4 +551,36 @@ public void setFiles(ArrayList> files) { this.files = files; } + public Long getNurseID() { + return nurseID; + } + + public void setNurseID(Long nurseID) { + this.nurseID = nurseID; + } + + public Long getDoctorID() { + return doctorID; + } + + public void setDoctorID(Long doctorID) { + this.doctorID = doctorID; + } + + public Long getPharmacistID() { + return pharmacistID; + } + + public void setPharmacistID(Long pharmacistID) { + this.pharmacistID = pharmacistID; + } + + public Long getLabTechnicianID() { + return labTechnicianID; + } + + public void setLabTechnicianID(Long labTechnicianID) { + this.labTechnicianID = labTechnicianID; + } + } diff --git a/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java b/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java index 6f6390ee..d14e3c3f 100644 --- a/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java +++ b/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java @@ -15,6 +15,9 @@ public interface UserLoginRepo extends CrudRepository { @Query(" SELECT u FROM Users u WHERE u.userID = :userID AND u.deleted = false ") public Users getUserByUserID(@Param("userID") Long userID); + @Query(" SELECT u FROM Users u WHERE u.userName = :UserName AND u.deleted = false ") + public Users getUserByUsername(@Param("UserName") String username); + @Query(nativeQuery = true,value = "select rolename from m_role where roleid in (select roleid from m_userservicerolemapping where userid=:userID)") List getRoleNamebyUserId(@Param("userID") Long userID); diff --git a/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java b/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java index c4a03286..75d075f9 100644 --- a/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java +++ b/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java @@ -101,4 +101,16 @@ public int updateFileID(@Param("fileIDs") String fileIDs, @Param("regID") Long r @Query(" UPDATE BeneficiaryVisitDetail set vanSerialNo = :benVisitID WHERE benVisitID = :benVisitID") int updateVanSerialNo(@Param("benVisitID") Long benVisitID); + // store responsible doctor's user ID against the visit + @Transactional + @Modifying + @Query("UPDATE BeneficiaryVisitDetail set doctorID = :doctorID where visitCode = :visitCode ") + public Integer updateDoctorID(@Param("doctorID") Long doctorID, @Param("visitCode") Long visitCode); + + // store responsible lab technician's user ID against the visit + @Transactional + @Modifying + @Query("UPDATE BeneficiaryVisitDetail set labTechnicianID = :labTechnicianID where visitCode = :visitCode ") + public Integer updateLabTechnicianID(@Param("labTechnicianID") Long labTechnicianID, @Param("visitCode") Long visitCode); + } diff --git a/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java b/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java index cab7a2d3..7302abc4 100644 --- a/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java @@ -115,6 +115,34 @@ public class CommonDoctorServiceImpl { @Autowired private CookieUtil cookieUtil; + @Autowired + private com.iemr.mmu.repo.nurse.BenVisitDetailRepo benVisitDetailRepo; + @Autowired + private com.iemr.mmu.repo.login.UserLoginRepo userLoginRepo; + + /** + * Resolve the numeric user ID of the responsible staff member from the username + * captured in createdBy. Returns null if it cannot be resolved so an unknown + * staff member never blocks the flow. + */ + private Long resolveUserId(String username) { + if (username == null || username.trim().isEmpty()) + return null; + com.iemr.mmu.data.login.Users user = userLoginRepo.getUserByUsername(username.trim()); + return user != null ? user.getUserID() : null; + } + + /** + * Store the responsible doctor's user ID on the visit record, taken from createdBy. + */ + private void storeDoctorIDOnVisit(CommonUtilityClass commonUtilityClass) { + if (commonUtilityClass == null || commonUtilityClass.getVisitCode() == null) + return; + Long doctorID = resolveUserId(commonUtilityClass.getCreatedBy()); + if (doctorID != null) + benVisitDetailRepo.updateDoctorID(doctorID, commonUtilityClass.getVisitCode()); + } + @Autowired public void setSnomedServiceImpl(SnomedServiceImpl snomedServiceImpl) { this.snomedServiceImpl = snomedServiceImpl; @@ -778,6 +806,9 @@ public int updateBenFlowtableAfterDocDataSave(CommonUtilityClass commonUtilityCl Long tmpBenVisitID = commonUtilityClass.getBenVisitID(); Long tmpbeneficiaryRegID = commonUtilityClass.getBeneficiaryRegID(); + // Store the responsible doctor's user ID against the visit + storeDoctorIDOnVisit(commonUtilityClass); + // checking if test is prescribed if (isTestPrescribed) { docFlag = (short) 2; @@ -840,6 +871,9 @@ public int updateBenFlowtableAfterDocDataUpdate(CommonUtilityClass commonUtility Long tmpBenVisitID = commonUtilityClass.getBenVisitID(); Long tmpbeneficiaryRegID = commonUtilityClass.getBeneficiaryRegID(); + // Store the responsible doctor's user ID against the visit + storeDoctorIDOnVisit(commonUtilityClass); + if (commonUtilityClass.getIsSpecialist() != null && commonUtilityClass.getIsSpecialist() == true) { if (isTestPrescribed) tcSpecialistFlag = (short) 2; diff --git a/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java b/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java index 72e3c8f2..a7efa9d4 100644 --- a/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java @@ -145,6 +145,22 @@ public class CommonNurseServiceImpl implements CommonNurseService { private Integer TMReferredWL; @Autowired private BenVisitDetailRepo benVisitDetailRepo; + + @Autowired + private com.iemr.mmu.repo.login.UserLoginRepo userLoginRepo; + + /** + * Resolve the numeric user ID of the responsible staff member from the username + * captured in createdBy. Returns null if it cannot be resolved so an unknown + * staff member never blocks the save. + */ + private Long resolveUserId(String username) { + if (username == null || username.trim().isEmpty()) + return null; + com.iemr.mmu.data.login.Users user = userLoginRepo.getUserByUsername(username.trim()); + return user != null ? user.getUserID() : null; + } + @Autowired private BenChiefComplaintRepo benChiefComplaintRepo; @Autowired @@ -258,6 +274,10 @@ public Long saveBeneficiaryVisitDetails(BeneficiaryVisitDetail beneficiaryVisitD } beneficiaryVisitDetail.setReportFilePath(sb.toString()); + // Store the responsible nurse's user ID (resolved from the createdBy username) + if (beneficiaryVisitDetail.getNurseID() == null) + beneficiaryVisitDetail.setNurseID(resolveUserId(beneficiaryVisitDetail.getCreatedBy())); + response = benVisitDetailRepo.save(beneficiaryVisitDetail); benVisitDetailRepo.updateVanSerialNo(response.getBenVisitID()); diff --git a/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java b/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java index 1026f63c..96f3b5c1 100644 --- a/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java @@ -47,6 +47,22 @@ public class LabTechnicianServiceImpl implements LabTechnicianService { private LabResultEntryRepo labResultEntryRepo; private CommonBenStatusFlowServiceImpl commonBenStatusFlowServiceImpl; + @Autowired + private com.iemr.mmu.repo.nurse.BenVisitDetailRepo benVisitDetailRepo; + @Autowired + private com.iemr.mmu.repo.login.UserLoginRepo userLoginRepo; + + /** + * Resolve the numeric user ID of the responsible staff member from the username + * captured in createdBy. Returns null if it cannot be resolved. + */ + private Long resolveUserId(String username) { + if (username == null || username.trim().isEmpty()) + return null; + com.iemr.mmu.data.login.Users user = userLoginRepo.getUserByUsername(username.trim()); + return user != null ? user.getUserID() : null; + } + @Autowired public void setCommonBenStatusFlowServiceImpl(CommonBenStatusFlowServiceImpl commonBenStatusFlowServiceImpl) { this.commonBenStatusFlowServiceImpl = commonBenStatusFlowServiceImpl; @@ -345,6 +361,12 @@ public Integer saveLabTestResult(JsonObject requestOBJ) throws Exception { labResultSaveFlag = saveLabTestResult(wrapperLabResults); if (labResultSaveFlag == 1) { + // Store the responsible lab technician's user ID against the visit + if (wrapperLabResults.getVisitCode() != null) { + Long labTechnicianID = resolveUserId(wrapperLabResults.getCreatedBy()); + if (labTechnicianID != null) + benVisitDetailRepo.updateLabTechnicianID(labTechnicianID, wrapperLabResults.getVisitCode()); + } int i = updateBenFlowStatusFlagAfterLabResultEntry(wrapperLabResults.getLabCompleted(), wrapperLabResults.getBenFlowID(), wrapperLabResults.getBeneficiaryRegID(), wrapperLabResults.getVisitID(), wrapperLabResults.getNurseFlag(), From 5b6bcf050f23e4c0aebbb1afef17c0329327b8a1 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 14 Jul 2026 16:17:10 +0530 Subject: [PATCH 02/81] fix: pom version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 990e3672..7a55041c 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.iemr.mmu mmu-api - 3.6.2 + 3.9.0 war MMU-API From c5b72c8a3d60e213c2335dca1f6a259b6fa87b83 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 17:33:25 +0530 Subject: [PATCH 03/81] feat(stoptb): add Nikshay ID Generator CSV export/import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a date-range-scoped export of Stop TB beneficiaries as a CSV formatted for the Nikshay ID Generator app, and an import endpoint to write the portal-generated Nikshay IDs back onto those beneficiaries. Scoped by visit-date range only, no vanID/servicePointID — MMU runs one local database per van, so everything in it already belongs to the current van/service point. Location columns (village/healthFacility/tu/district/state) are resolved against Nikshay's own, isolated location hierarchy (m_nikshay_village -> m_nikshay_facility -> m_nikshay_tu -> m_nikshay_district -> m_nikshay_state), not AMRIT's standard masters — a Stop TB beneficiary's I_bendemographics.DistrictBranchID actually holds their Nikshay Village ID (the same overload Common-API's NikshayAddressResolver relies on), so every other location field is derived by walking up that one chain. A beneficiary whose village doesn't resolve all the way to a state is silently left out of the export, same as one who already has a Nikshay ID. New endpoints: - GET /stopTb/nikshay/exportBeneficiariesCsv?fromDate=&toDate= - POST /stopTb/nikshay/importResultsCsv?visitDate=&file= Co-Authored-By: Claude Sonnet 5 --- pom.xml | 6 + .../stoptb/NikshayExportController.java | 162 ++++++++++++ .../repo/stoptb/NikshayExportRepository.java | 226 ++++++++++++++++ .../service/stoptb/NikshayExportService.java | 249 ++++++++++++++++++ .../service/stoptb/NikshayImportService.java | 155 +++++++++++ 5 files changed, 798 insertions(+) create mode 100644 src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java create mode 100644 src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java create mode 100644 src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java create mode 100644 src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java diff --git a/pom.xml b/pom.xml index 990e3672..59ea5dce 100644 --- a/pom.xml +++ b/pom.xml @@ -292,6 +292,12 @@ h2 runtime + + + org.apache.commons + commons-csv + 1.11.0 + ${project.artifactId}-${project.version} diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java new file mode 100644 index 00000000..7757528c --- /dev/null +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -0,0 +1,162 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.controller.stoptb; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import com.iemr.mmu.service.stoptb.NikshayExportService; +import com.iemr.mmu.service.stoptb.NikshayImportService; +import com.iemr.mmu.service.stoptb.NikshayImportService.ImportSummary; +import com.iemr.mmu.utils.JwtUtil; + +import io.swagger.v3.oas.annotations.Operation; +import jakarta.servlet.http.HttpServletRequest; + +/** + * Downloads/uploads the Stop TB Nikshay ID Generator CSV. + * + * Scoped only by visit-date range — MMU runs one local database per van, so + * everything in it already belongs to the current van/service point; there + * is deliberately no vanID/servicePointID parameter on either endpoint. + */ +@RestController +@RequestMapping(value = "/stopTb/nikshay", headers = "Authorization") +@PreAuthorize("hasRole('NURSE') || hasRole('PHARMACIST') || hasRole('LABTECHNICIAN') || hasRole('DOCTOR') || hasRole('LAB_TECHNICIAN') || hasRole('TC_SPECIALIST') || hasRole('ONCOLOGIST') || hasRole('RADIOLOGIST')") +public class NikshayExportController { + private static final Logger logger = LoggerFactory.getLogger(NikshayExportController.class); + private static final DateTimeFormatter DATE_FMT = DateTimeFormatter.ISO_LOCAL_DATE; + + @Autowired + private NikshayExportService nikshayExportService; + + @Autowired + private NikshayImportService nikshayImportService; + + @Autowired + private JwtUtil jwtUtil; + + /** Best-effort — this is only used for a created_by/modified_by audit column, + * never for authorization (the filter chain/PreAuthorize already handled that). */ + private String currentUsername(HttpServletRequest request) { + try { + String header = request.getHeader("Authorization"); + if (header == null) { + return "unknown"; + } + String token = header.startsWith("Bearer ") ? header.substring(7) : header; + String username = jwtUtil.extractUsername(token); + return username != null ? username : "unknown"; + } catch (Exception e) { + return "unknown"; + } + } + + @Operation(summary = "Download Stop TB beneficiaries for a date range as a CSV formatted for the Nikshay ID Generator") + @GetMapping(value = "/exportBeneficiariesCsv") + public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String fromDateStr, + @RequestParam("toDate") String toDateStr) { + + LocalDate fromDate; + LocalDate toDate; + try { + fromDate = LocalDate.parse(fromDateStr, DATE_FMT); + toDate = LocalDate.parse(toDateStr, DATE_FMT); + } catch (DateTimeParseException e) { + return ResponseEntity.badRequest().body("fromDate/toDate must be in YYYY-MM-DD format"); + } + if (toDate.isBefore(fromDate)) { + return ResponseEntity.badRequest().body("toDate must be on or after fromDate"); + } + + int excludedAlreadyGenerated; + int excludedUnresolvedLocation; + try { + excludedAlreadyGenerated = nikshayExportService.countAlreadyGenerated(fromDate, toDate); + excludedUnresolvedLocation = nikshayExportService.countUnresolvedLocation(fromDate, toDate); + } catch (Exception e) { + logger.error("Error preparing Nikshay beneficiary export", e); + return ResponseEntity.status(500).body("Could not prepare the export"); + } + + StreamingResponseBody body = outputStream -> { + try { + nikshayExportService.streamBeneficiariesCsv(fromDate, toDate, outputStream); + } catch (Exception e) { + // The HTTP status/headers are already committed by the time streaming + // starts, so a mid-stream failure can only be logged, not surfaced + // as a clean error response. + logger.error("Error streaming Nikshay beneficiary CSV", e); + } + }; + + String filename = "nikshay-beneficiaries-" + fromDate + "-to-" + toDate + ".csv"; + return ResponseEntity.ok().contentType(MediaType.parseMediaType("text/csv")) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"") + .header("X-Excluded-Existing-Nikshay-Id-Count", String.valueOf(excludedAlreadyGenerated)) + .header("X-Excluded-Unresolved-Location-Count", String.valueOf(excludedUnresolvedLocation)) + .body(body); + } + + @Operation(summary = "Upload the Nikshay ID Generator app's results CSV to write generated Nikshay IDs " + + "back onto the beneficiaries — each row is matched by its own benRegId column, " + + "a pass-through field the export added that the ID Generator app never touches") + @PostMapping(value = "/importResultsCsv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity importResultsCsv(@RequestParam("visitDate") String visitDateStr, + @RequestParam("file") MultipartFile file, HttpServletRequest request) { + if (file == null || file.isEmpty()) { + return ResponseEntity.badRequest().body("A results CSV file is required"); + } + LocalDate visitDate; + try { + visitDate = LocalDate.parse(visitDateStr, DATE_FMT); + } catch (DateTimeParseException e) { + return ResponseEntity.badRequest().body("visitDate must be in YYYY-MM-DD format"); + } + try { + ImportSummary summary = nikshayImportService.importResults(visitDate, file.getInputStream(), + currentUsername(request)); + return ResponseEntity.ok(summary); + } catch (IllegalArgumentException e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } catch (Exception e) { + logger.error("Error importing Nikshay results CSV", e); + return ResponseEntity.status(500).body("Could not import the results file"); + } + } +} diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java new file mode 100644 index 00000000..4ab3ff13 --- /dev/null +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -0,0 +1,226 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.repo.stoptb; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.function.Consumer; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementSetter; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; +import org.springframework.stereotype.Repository; + +/** + * Reads Stop TB beneficiaries for the Nikshay ID Generator CSV export. + * + * MMU runs one local database per van, so a beneficiary visible here already + * belongs to the current van/service point by construction — there is + * deliberately no vanID/servicePointID filter, only a visit-date range. + * + * Location columns (village/healthFacility/tu/district/state) are resolved + * against Nikshay's own, isolated location hierarchy — m_nikshay_village → + * m_nikshay_facility → m_nikshay_tu → m_nikshay_district → m_nikshay_state — + * not AMRIT's standard masters. A Stop TB beneficiary's own + * I_bendemographics.DistrictBranchID holds their Nikshay Village ID (the same + * overload Common-API's NikshayAddressResolver relies on for the + * beneficiary-detail API), so every other location field is derived by + * walking up that one chain rather than trusting separately-stored + * block/district fields. + * + * Beneficiaries are silently left out of the streamed rows (see + * streamPendingBeneficiaries) in two cases — already having a Nikshay ID + * recorded, or their village not resolving all the way up to a state. Both + * are counted so callers can report totals, but there is deliberately no + * separate report of *who* was skipped or why. + */ +@Repository +public class NikshayExportRepository { + + @Autowired + private DataSource dataSource; + + private JdbcTemplate getJdbcTemplate() { + return new JdbcTemplate(dataSource); + } + + /** One beneficiary's raw, unmapped source data — Nikshay-vocabulary + * mapping/validation happens in the service layer, not here. benRegId is + * carried into the CSV itself (as a pass-through column the Nikshay ID + * Generator app never touches) so results can be matched back to a + * beneficiary on import without needing any AMRIT-side row tracking. */ + public record NikshayRawRow(Long benRegId, String firstName, String middleLastName, Integer age, String gender, + String phone, String address, String stateName, String districtName, String tu, String healthFacility, + String village, String pincode, String maritalStatus, String caste, String occupation, + String socioeconomicStatus, String chiefComplaint, String hivStatus, Boolean isHivPos) { + } + + // Placeholders in order: [1] fromDate (inclusive), [2] toDate-exclusive-upper-bound. + private static final String BASE_SELECT = "SELECT " + + " b.BeneficiaryRegID AS benRegId, " + + " b.FirstName AS firstName, " + + " TRIM(CONCAT(COALESCE(b.MiddleName,''),' ',COALESCE(b.LastName,''))) AS middleLastName, " + + " TIMESTAMPDIFF(YEAR, b.DOB, CURDATE()) AS age, " + + " g.GenderName AS gender, " + + " (SELECT p.PhoneNo FROM i_benphonemap p WHERE p.BenificiaryRegID = b.BeneficiaryRegID " + + " AND p.Deleted = 0 ORDER BY p.BenPhMapID ASC LIMIT 1) AS phone, " + + " TRIM(CONCAT_WS(', ', d.AddressLine1, d.AddressLine2, d.AddressLine3, d.AddressLine4, d.AddressLine5)) AS address, " + + " ns.StateName AS stateName, " + + " nd.DistrictName AS districtName, " + + " ntu.TUName AS tu, " + + " nf.FacilityName AS healthFacility, " + + " nv.VillageName AS village, " + + " d.PinCode AS pincode, " + + " ms.Status AS maritalStatus, " + + " c.CommunityType AS caste, " + + " occ.OccupationType AS occupation, " + + " inc.IncomeStatus AS socioeconomicStatus, " + + " (SELECT o.chief_complaint FROM tb_stoptb_general_opd o WHERE o.ben_reg_id = b.BeneficiaryRegID " + + " AND o.deleted = 0 ORDER BY o.id DESC LIMIT 1) AS chiefComplaint, " + + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = b.BeneficiaryRegID " + + " AND ge.deleted = 0 ORDER BY ge.id DESC LIMIT 1) AS hivStatus, " + + " b.IsHIVPos AS isHivPos, " + + " (SELECT diag.nikshay_id FROM tb_stoptb_diagnostics diag WHERE diag.ben_reg_id = b.BeneficiaryRegID " + + " AND diag.nikshay_id IS NOT NULL AND diag.deleted = 0 ORDER BY diag.id DESC LIMIT 1) AS existingNikshayId " + + "FROM i_beneficiary b " + + "LEFT JOIN I_bendemographics d ON d.BeneficiaryRegID = b.BeneficiaryRegID " + + "LEFT JOIN m_gender g ON g.GenderID = b.GenderID " + + "LEFT JOIN m_maritalstatus ms ON ms.MaritalStatusID = b.MaritalStatusID " + + "LEFT JOIN m_community c ON c.CommunityID = d.CommunityID " + + "LEFT JOIN m_beneficiaryincomestatus inc ON inc.IncomeStatusID = d.IncomeStatusID " + + "LEFT JOIN m_beneficiaryoccupation occ ON occ.OccupationID = d.OccupationID " + // Stop TB's DistrictBranchID is overloaded to hold the Nikshay Village ID + // (see class Javadoc) — walk Nikshay's own hierarchy up from there. + + "LEFT JOIN m_nikshay_village nv ON nv.NikshayVillageID = d.DistrictBranchID AND nv.Deleted = 0 " + + "LEFT JOIN m_nikshay_facility nf ON nf.NikshayFacilityID = nv.NikshayFacilityID AND nf.Deleted = 0 " + + "LEFT JOIN m_nikshay_tu ntu ON ntu.NikshayTUID = nf.NikshayTUID AND ntu.Deleted = 0 " + + "LEFT JOIN m_nikshay_district nd ON nd.NikshayDistrictID = ntu.NikshayDistrictID AND nd.Deleted = 0 " + + "LEFT JOIN m_nikshay_state ns ON ns.NikshayStateID = nd.NikshayStateID AND ns.Deleted = 0 " + + "WHERE b.Deleted = 0 " + + " AND b.BeneficiaryRegID IN ( " + + " SELECT DISTINCT v.beneficiary_reg_id FROM tb_stoptb_visit v " + + " WHERE v.visit_date >= ? AND v.visit_date < ? " + + " )"; + + public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { + String sql = "SELECT COUNT(*) FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NOT NULL"; + Integer count = getJdbcTemplate().query(sql, pss(fromDate, toDate), rs -> rs.next() ? rs.getInt(1) : 0); + return count == null ? 0 : count; + } + + public int countUnresolvedLocation(LocalDate fromDate, LocalDate toDate) { + String sql = "SELECT COUNT(*) FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NULL " + + "AND (t.village IS NULL OR t.healthFacility IS NULL OR t.tu IS NULL " + + "OR t.districtName IS NULL OR t.stateName IS NULL)"; + Integer count = getJdbcTemplate().query(sql, pss(fromDate, toDate), rs -> rs.next() ? rs.getInt(1) : 0); + return count == null ? 0 : count; + } + + /** Streams every not-yet-Nikshay-ID'd beneficiary in the date range to + * {@code rowConsumer} one row at a time, without materializing the full + * result set in memory — safe for large date ranges. Silently skips any + * beneficiary whose Nikshay village doesn't resolve all the way up to a + * state (see class Javadoc). */ + public void streamPendingBeneficiaries(LocalDate fromDate, LocalDate toDate, Consumer rowConsumer) { + String sql = "SELECT * FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NULL " + + "AND t.village IS NOT NULL AND t.healthFacility IS NOT NULL AND t.tu IS NOT NULL " + + "AND t.districtName IS NOT NULL AND t.stateName IS NOT NULL"; + JdbcTemplate jdbcTemplate = getJdbcTemplate(); + // MySQL Connector/J-specific: Integer.MIN_VALUE forces true row-by-row + // network streaming instead of buffering the whole result set client-side. + jdbcTemplate.setFetchSize(Integer.MIN_VALUE); + jdbcTemplate.query(sql, pss(fromDate, toDate), (ResultSet rs) -> rowConsumer.accept(mapRow(rs))); + } + + private PreparedStatementSetter pss(LocalDate fromDate, LocalDate toDate) { + return (PreparedStatement ps) -> { + ps.setTimestamp(1, Timestamp.valueOf(fromDate.atStartOfDay())); + ps.setTimestamp(2, Timestamp.valueOf(toDate.plusDays(1).atStartOfDay())); + }; + } + + private NikshayRawRow mapRow(ResultSet rs) throws SQLException { + return new NikshayRawRow( + rs.getObject("benRegId", Long.class), + rs.getString("firstName"), + rs.getString("middleLastName"), + rs.getObject("age", Integer.class), + rs.getString("gender"), + rs.getString("phone"), + rs.getString("address"), + rs.getString("stateName"), + rs.getString("districtName"), + rs.getString("tu"), + rs.getString("healthFacility"), + rs.getString("village"), + rs.getString("pincode"), + rs.getString("maritalStatus"), + rs.getString("caste"), + rs.getString("occupation"), + rs.getString("socioeconomicStatus"), + rs.getString("chiefComplaint"), + rs.getString("hivStatus"), + rs.getObject("isHivPos", Boolean.class)); + } + + /** The most recent tb_stoptb_diagnostics row for this beneficiary, if any — + * looked up live at import time (no export-time snapshot needed, since the + * beneficiary is identified directly from the results CSV's own benRegId + * column). Null if none exists yet. */ + public Long findLatestDiagnosticsId(Long benRegId) { + String sql = "SELECT id FROM tb_stoptb_diagnostics WHERE ben_reg_id = ? AND deleted = 0 " + + "ORDER BY id DESC LIMIT 1"; + return getJdbcTemplate().query(sql, (ResultSet rs) -> rs.next() ? rs.getLong("id") : null, benRegId); + } + + public void updateNikshayId(Long diagnosticsId, String nikshayId, String modifiedBy) { + String sql = "UPDATE tb_stoptb_diagnostics SET nikshay_id = ?, modified_by = ?, " + + "last_mod_date = CURRENT_TIMESTAMP WHERE id = ?"; + getJdbcTemplate().update(sql, nikshayId, modifiedBy, diagnosticsId); + } + + /** Called when a beneficiary had no tb_stoptb_diagnostics row yet — creates + * one to hold the Nikshay ID the portal generated. */ + public Long insertDiagnosticsWithNikshayId(Long benRegId, LocalDate visitDate, String nikshayId, + String createdBy) { + String sql = "INSERT INTO tb_stoptb_diagnostics (ben_reg_id, visit_date, nikshay_id, created_by) " + + "VALUES (?, ?, ?, ?)"; + KeyHolder keyHolder = new GeneratedKeyHolder(); + getJdbcTemplate().update(connection -> { + PreparedStatement ps = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); + ps.setLong(1, benRegId); + ps.setTimestamp(2, Timestamp.valueOf(visitDate.atStartOfDay())); + ps.setString(3, nikshayId); + ps.setString(4, createdBy); + return ps; + }, keyHolder); + return keyHolder.getKey().longValue(); + } +} diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java new file mode 100644 index 00000000..44078859 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java @@ -0,0 +1,249 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.stoptb; + +import java.io.BufferedWriter; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iemr.mmu.repo.stoptb.NikshayExportRepository; +import com.iemr.mmu.repo.stoptb.NikshayExportRepository.NikshayRawRow; + +/** + * Builds the Nikshay ID Generator CSV for Stop TB beneficiaries. + * + * Column order/names are fixed by the Nikshay ID Generator app's own import + * format — every categorical field below resolves to one of its allowed + * values (case-insensitive match) when AMRIT's data lines up, otherwise a + * safe always-valid fallback (e.g. "Unknown"), never a guessed/fuzzy mapping + * onto the wrong specific label. + * + * state/district/tu/healthFacility/village are all resolved against + * Nikshay's own location hierarchy (m_nikshay_state/district/tu/facility/ + * village), not AMRIT's standard masters — see NikshayExportRepository's + * class Javadoc for how that chain is walked from the beneficiary's own + * saved Nikshay village. A beneficiary whose village doesn't resolve all the + * way up that chain never reaches this class — the repository leaves such + * rows out of the stream entirely, silently, the same way it already + * silently skips beneficiaries who already have a Nikshay ID. + * + * Known gaps, best-effort until resolved elsewhere: + * - occupation/area: no reliable AMRIT-to-Nikshay label mapping exists yet, + * so these always fall back to "Unknown" (a valid value for both). + * - symptoms: AMRIT has no structured Stop TB symptom checklist wired up yet + * (that data lives in the generic Dynamic Form response tables, whose + * question mapping isn't resolved). Best-effort: "Asymptomatic" when no + * chief complaint was recorded, "Others" otherwise. + * - gender has no safe generic fallback (no "Unknown" option in Nikshay's + * 3-value list) — left blank on an unmapped value rather than guessed, + * which will surface as a clear per-row error in the ID Generator app. + * - typeOfCaseFinding: no Active/Passive signal in AMRIT yet — always + * "Passive", matching the portal's own default. + */ +@Service +public class NikshayExportService { + + // benRegId is a pass-through column, not one of Nikshay's own template fields — the ID + // Generator app never reads or displays it, but carries it straight through to the + // results file, which is how an uploaded results CSV gets matched back to a beneficiary. + private static final String[] CSV_HEADER = { "benRegId", "typeOfCaseFinding", "caste", "firstName", + "middleLastName", "age", "gender", "primaryPhone", "address", "state", "district", "tu", "healthFacility", + "village", "pincode", "area", "maritalStatus", "occupation", "socioeconomicStatus", "symptoms", + "hivStatus" }; + + private static final Set GENDER_VALUES = setOf("Male", "Female", "Transgender"); + private static final Set CASTE_VALUES = setOf("SC", "ST", "Other"); + private static final Set MARITAL_VALUES = setOf("Single", "Married", "Unknown"); + private static final Set SOCIOECONOMIC_VALUES = setOf("APL", "BPL", "Unknown"); + private static final Set HIV_VALUES = setOf("Positive", "Reactive", "Non Reactive / Negative", "Unknown"); + // Verbatim from the ID Generator's own allowed-values list — spellings/typos + // are copied exactly as the portal defines them. + private static final Set OCCUPATION_VALUES = setOf("Legislators and Senior officials", + "Corporate Manager", "General Manager", + "Physical, mathematical and engineering science professional", + "Life sciences and health professional", "Teaching professional", "Other professional", "Office Clerk", + "Customer Services Clerks", "Personal Protective Service Providers", + "Models, Sales Persons and Demonstrators", "Market oriented skilled agriculutre and fishery workers", + "Subsitence agriculture and fishery workers", "Extraction and building trade workers", + "Metal, Machinery and related trades workers", + "Precision, handicraft, printing and related trade workers", + "Other Craft and related traders and workers", "Stationary Plant and related Operators", + "Machine Operators and Assembler", "Drivers and Mobile Plant Operators", + "Sales and Services elementry occupations", "Agriculture, fishery and related labour", + "Laborers in mining, construction, manufecturing and transport", "New Workers seeking employment", + "Workers reporting occupation unidentifiable or inadequately", "Workers no reporting any occupation", + "House Wife", "Unknown"); + + @Autowired + private NikshayExportRepository nikshayExportRepository; + + private static Set setOf(String... values) { + return new HashSet<>(Arrays.asList(values)); + } + + public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { + return nikshayExportRepository.countAlreadyGenerated(fromDate, toDate); + } + + public int countUnresolvedLocation(LocalDate fromDate, LocalDate toDate) { + return nikshayExportRepository.countUnresolvedLocation(fromDate, toDate); + } + + /** Writes the CSV (header + one row per pending beneficiary) directly to + * {@code outputStream} as rows arrive from the database — never buffers + * the whole file in memory. Caller owns closing {@code outputStream}. */ + public void streamBeneficiariesCsv(LocalDate fromDate, LocalDate toDate, OutputStream outputStream) { + Writer writer = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)); + try { + writer.write(String.join(",", CSV_HEADER)); + writer.write("\r\n"); + + nikshayExportRepository.streamPendingBeneficiaries(fromDate, toDate, row -> { + try { + writer.write(toCsvLine(row)); + writer.write("\r\n"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + writer.flush(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private String toCsvLine(NikshayRawRow row) { + String[] values = { + String.valueOf(row.benRegId()), + "Passive", + matchOrDefault(row.caste(), CASTE_VALUES, "Other"), + nullToEmpty(row.firstName()), + nullToEmpty(row.middleLastName()), + ageOrBlank(row.age()), + matchOrBlank(row.gender(), GENDER_VALUES), + validPhoneOrBlank(row.phone()), + nullToEmpty(row.address()), + nullToEmpty(row.stateName()), + nullToEmpty(row.districtName()), + nullToEmpty(row.tu()), + nullToEmpty(row.healthFacility()), + nullToEmpty(row.village()), + validPincodeOrBlank(row.pincode()), + "Unknown", // area - no reliable AMRIT-to-Nikshay mapping available yet + matchOrDefault(row.maritalStatus(), MARITAL_VALUES, "Unknown"), + matchOrDefault(row.occupation(), OCCUPATION_VALUES, "Unknown"), + matchOrDefault(row.socioeconomicStatus(), SOCIOECONOMIC_VALUES, "Unknown"), + symptomsFrom(row.chiefComplaint()), + hivStatusFrom(row.hivStatus(), row.isHivPos()), + }; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < values.length; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(csvEscape(values[i])); + } + return sb.toString(); + } + + private static String nullToEmpty(String s) { + return s == null ? "" : s.trim(); + } + + private static String matchOrBlank(String raw, Set allowed) { + if (raw == null) { + return ""; + } + String trimmed = raw.trim(); + for (String candidate : allowed) { + if (candidate.equalsIgnoreCase(trimmed)) { + return candidate; + } + } + return ""; + } + + private static String matchOrDefault(String raw, Set allowed, String fallback) { + String matched = matchOrBlank(raw, allowed); + return matched.isEmpty() ? fallback : matched; + } + + private static String ageOrBlank(Integer age) { + return (age != null && age >= 1 && age <= 99) ? String.valueOf(age) : ""; + } + + private static String validPhoneOrBlank(String raw) { + if (raw == null) { + return ""; + } + String digits = raw.replaceAll("[^0-9]", ""); + if (digits.length() > 10) { + digits = digits.substring(digits.length() - 10); + } + return digits.matches("[1-9][0-9]{9}") ? digits : ""; + } + + private static String validPincodeOrBlank(String raw) { + if (raw == null) { + return ""; + } + String trimmed = raw.trim(); + return trimmed.matches("[0-9]{6}") ? trimmed : ""; + } + + private static String symptomsFrom(String chiefComplaint) { + return (chiefComplaint == null || chiefComplaint.trim().isEmpty()) ? "Asymptomatic" : "Others"; + } + + private static String hivStatusFrom(String rawHivStatus, Boolean isHivPos) { + String matched = matchOrBlank(rawHivStatus, HIV_VALUES); + if (!matched.isEmpty()) { + return matched; + } + if (Boolean.TRUE.equals(isHivPos)) { + return "Positive"; + } + if (Boolean.FALSE.equals(isHivPos)) { + return "Non Reactive / Negative"; + } + return "Unknown"; + } + + private static String csvEscape(String value) { + if (value == null || value.isEmpty()) { + return ""; + } + boolean needsQuoting = value.indexOf(',') >= 0 || value.indexOf('"') >= 0 || value.indexOf('\n') >= 0 + || value.indexOf('\r') >= 0; + String escaped = value.replace("\"", "\"\""); + return needsQuoting ? "\"" + escaped + "\"" : escaped; + } +} diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java new file mode 100644 index 00000000..0fb78054 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -0,0 +1,155 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.stoptb; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.iemr.mmu.repo.stoptb.NikshayExportRepository; + +/** + * Imports the Nikshay ID Generator desktop app's results CSV, writing the + * portal-generated Nikshay IDs back onto the right beneficiaries. + * + * Each row is identified by its own {@code benRegId} column — a pass-through + * field the exported CSV carries that isn't one of Nikshay's own template + * columns, so the ID Generator app never touches it but does carry it + * straight through to the results file. That means no AMRIT-side row/order + * tracking is needed: each result row is self-identifying, and the + * beneficiary's tb_stoptb_diagnostics row (if any) is resolved live at + * import time. No vanID/servicePointID scoping either — same as export, + * MMU's per-van local database makes that implicit. + * + * Row status handling: + * - "success": generatedId is the new Nikshay ID — written as-is. + * - "skipped" (portal-detected duplicate): generatedId is one or more + * existing patient IDs, space-separated. A single ID is written the same + * as a success; more than one is ambiguous and left for manual review + * rather than guessed. + * - "failed": never written; surfaced in the response for visibility. + */ +@Service +public class NikshayImportService { + + private static final List REQUIRED_COLUMNS = List.of("benRegId", "firstName", "middleLastName", + "generatedId", "status"); + + public record ImportRowResult(int rowIndex, Long benRegId, String firstName, String middleLastName, + String status, String generatedId, String note) { + } + + public record ImportSummary(int csvRowCount, int updated, int failed, int needsReview, + List needsReviewRows, List failedRows) { + } + + @Autowired + private NikshayExportRepository nikshayExportRepository; + + public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStream, String modifiedBy) + throws Exception { + List records; + boolean hasErrorColumn; + CSVFormat format = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).setTrim(true).build(); + try (CSVParser parser = new CSVParser(new InputStreamReader(csvInputStream, StandardCharsets.UTF_8), + format)) { + Map header = parser.getHeaderMap(); + for (String required : REQUIRED_COLUMNS) { + if (!header.containsKey(required)) { + throw new IllegalArgumentException("Results CSV is missing required column: " + required); + } + } + hasErrorColumn = header.containsKey("error"); + records = parser.getRecords(); + } + + int updated = 0; + List needsReview = new ArrayList<>(); + List failedRows = new ArrayList<>(); + + for (int i = 0; i < records.size(); i++) { + CSVRecord record = records.get(i); + String firstName = record.get("firstName"); + String middleLastName = record.get("middleLastName"); + String status = record.get("status").trim(); + String generatedId = record.get("generatedId").trim(); + + Long benRegId = parseBenRegId(record.get("benRegId")); + if (benRegId == null) { + failedRows.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, + "Row has a missing/invalid benRegId — was this file exported by AMRIT?")); + continue; + } + + if ("success".equalsIgnoreCase(status) || "skipped".equalsIgnoreCase(status)) { + String[] tokens = generatedId.isEmpty() ? new String[0] : generatedId.split("\\s+"); + if (tokens.length == 1) { + writeNikshayId(visitDate, benRegId, tokens[0], modifiedBy); + updated++; + } else { + String note = tokens.length == 0 ? "Row marked " + status + " but has no generatedId." + : "Multiple possible existing Nikshay IDs (" + generatedId + + ") — needs manual confirmation."; + needsReview.add( + new ImportRowResult(i, benRegId, firstName, middleLastName, status, generatedId, note)); + } + } else { + String error = hasErrorColumn ? record.get("error") : ""; + failedRows.add( + new ImportRowResult(i, benRegId, firstName, middleLastName, status, generatedId, error)); + } + } + + return new ImportSummary(records.size(), updated, failedRows.size(), needsReview.size(), needsReview, + failedRows); + } + + private static Long parseBenRegId(String raw) { + if (raw == null || raw.isBlank()) { + return null; + } + try { + return Long.valueOf(raw.trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private void writeNikshayId(LocalDate visitDate, Long benRegId, String nikshayId, String modifiedBy) { + Long diagnosticsId = nikshayExportRepository.findLatestDiagnosticsId(benRegId); + if (diagnosticsId != null) { + nikshayExportRepository.updateNikshayId(diagnosticsId, nikshayId, modifiedBy); + } else { + nikshayExportRepository.insertDiagnosticsWithNikshayId(benRegId, visitDate, nikshayId, modifiedBy); + } + } +} From de0b7bce4c51bcccbb76dc8005b1a65e4da7b13b Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 19:32:15 +0530 Subject: [PATCH 04/81] fix(stoptb): store Nikshay ID on tb_suspected, match import rows by phone+name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections after checking real data: 1. nikshay_id now lives on tb_suspected, not tb_stoptb_diagnostics — both tables have the column, but tb_suspected is the one this feature is meant to use going forward. Both the export's already-has-an-ID check and the import's write path now target tb_suspected.benRegID. 2. A real Nikshay ID Generator results file has no beneficiary ID column of any kind — only the original template columns plus its own generatedId/status/durationSec/error. The import can no longer assume benRegId round-trips through untouched. Rows are now matched back to a beneficiary by normalized primaryPhone + case-insensitive firstName (NikshayExportRepository.findMatchingBeneficiaryIds); anything other than exactly one match is left for manual review rather than guessed. Co-Authored-By: Claude Sonnet 5 --- .../repo/stoptb/NikshayExportRepository.java | 52 ++++++++++----- .../service/stoptb/NikshayImportService.java | 64 ++++++++++++------- 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 4ab3ff13..d6567cfe 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -27,6 +27,7 @@ import java.sql.Statement; import java.sql.Timestamp; import java.time.LocalDate; +import java.util.List; import java.util.function.Consumer; import javax.sql.DataSource; @@ -60,6 +61,10 @@ * recorded, or their village not resolving all the way up to a state. Both * are counted so callers can report totals, but there is deliberately no * separate report of *who* was skipped or why. + * + * The Nikshay ID itself lives on tb_suspected.nikshay_id — not + * tb_stoptb_diagnostics, which also has a nikshay_id column but is not the + * table this feature writes to. */ @Repository public class NikshayExportRepository { @@ -107,8 +112,8 @@ public record NikshayRawRow(Long benRegId, String firstName, String middleLastNa + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = b.BeneficiaryRegID " + " AND ge.deleted = 0 ORDER BY ge.id DESC LIMIT 1) AS hivStatus, " + " b.IsHIVPos AS isHivPos, " - + " (SELECT diag.nikshay_id FROM tb_stoptb_diagnostics diag WHERE diag.ben_reg_id = b.BeneficiaryRegID " - + " AND diag.nikshay_id IS NOT NULL AND diag.deleted = 0 ORDER BY diag.id DESC LIMIT 1) AS existingNikshayId " + + " (SELECT s.nikshay_id FROM tb_suspected s WHERE s.benRegID = b.BeneficiaryRegID " + + " AND s.nikshay_id IS NOT NULL ORDER BY s.id DESC LIMIT 1) AS existingNikshayId " + "FROM i_beneficiary b " + "LEFT JOIN I_bendemographics d ON d.BeneficiaryRegID = b.BeneficiaryRegID " + "LEFT JOIN m_gender g ON g.GenderID = b.GenderID " @@ -190,28 +195,43 @@ private NikshayRawRow mapRow(ResultSet rs) throws SQLException { rs.getObject("isHivPos", Boolean.class)); } - /** The most recent tb_stoptb_diagnostics row for this beneficiary, if any — - * looked up live at import time (no export-time snapshot needed, since the + /** Finds beneficiaries matching a results-file row by content, since the + * real Nikshay ID Generator app's results CSV carries no beneficiary ID of + * any kind back — only phone/name/age survive the round trip. Matches on + * normalized 10-digit phone plus a case-insensitive first-name match; + * callers must treat anything other than exactly one result as ambiguous + * (e.g. a shared family phone number) rather than guessing. */ + public List findMatchingBeneficiaryIds(String phoneDigits, String firstName) { + String sql = "SELECT DISTINCT b.BeneficiaryRegID FROM i_beneficiary b " + + "JOIN i_benphonemap p ON p.BenificiaryRegID = b.BeneficiaryRegID AND p.Deleted = 0 " + + "WHERE b.Deleted = 0 AND p.PhoneNo = ? AND LOWER(TRIM(b.FirstName)) = LOWER(TRIM(?))"; + return getJdbcTemplate().query(sql, (rs, rowNum) -> rs.getLong("BeneficiaryRegID"), phoneDigits, firstName); + } + + /** The most recent tb_suspected row for this beneficiary, if any — looked + * up live at import time (no export-time snapshot needed, since the * beneficiary is identified directly from the results CSV's own benRegId - * column). Null if none exists yet. */ - public Long findLatestDiagnosticsId(Long benRegId) { - String sql = "SELECT id FROM tb_stoptb_diagnostics WHERE ben_reg_id = ? AND deleted = 0 " - + "ORDER BY id DESC LIMIT 1"; + * column). Null if none exists yet. Note: tb_suspected has no `deleted` + * column, unlike most other AMRIT tables. */ + public Long findLatestSuspectedId(Long benRegId) { + String sql = "SELECT id FROM tb_suspected WHERE benRegID = ? ORDER BY id DESC LIMIT 1"; return getJdbcTemplate().query(sql, (ResultSet rs) -> rs.next() ? rs.getLong("id") : null, benRegId); } - public void updateNikshayId(Long diagnosticsId, String nikshayId, String modifiedBy) { - String sql = "UPDATE tb_stoptb_diagnostics SET nikshay_id = ?, modified_by = ?, " + public void updateNikshayId(Long suspectedId, String nikshayId, String modifiedBy) { + String sql = "UPDATE tb_suspected SET nikshay_id = ?, modified_by = ?, " + "last_mod_date = CURRENT_TIMESTAMP WHERE id = ?"; - getJdbcTemplate().update(sql, nikshayId, modifiedBy, diagnosticsId); + getJdbcTemplate().update(sql, nikshayId, modifiedBy, suspectedId); } - /** Called when a beneficiary had no tb_stoptb_diagnostics row yet — creates - * one to hold the Nikshay ID the portal generated. */ - public Long insertDiagnosticsWithNikshayId(Long benRegId, LocalDate visitDate, String nikshayId, + /** Called when a beneficiary had no tb_suspected row yet — creates one to + * hold the Nikshay ID the portal generated. created_date is set explicitly + * because, unlike created_date on most other AMRIT tables, tb_suspected's + * has no DB-side default. */ + public Long insertSuspectedWithNikshayId(Long benRegId, LocalDate visitDate, String nikshayId, String createdBy) { - String sql = "INSERT INTO tb_stoptb_diagnostics (ben_reg_id, visit_date, nikshay_id, created_by) " - + "VALUES (?, ?, ?, ?)"; + String sql = "INSERT INTO tb_suspected (benRegID, visit_date, nikshay_id, created_by, created_date) " + + "VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)"; KeyHolder keyHolder = new GeneratedKeyHolder(); getJdbcTemplate().update(connection -> { PreparedStatement ps = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java index 0fb78054..36033dc6 100644 --- a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -41,14 +41,20 @@ * Imports the Nikshay ID Generator desktop app's results CSV, writing the * portal-generated Nikshay IDs back onto the right beneficiaries. * - * Each row is identified by its own {@code benRegId} column — a pass-through - * field the exported CSV carries that isn't one of Nikshay's own template - * columns, so the ID Generator app never touches it but does carry it - * straight through to the results file. That means no AMRIT-side row/order - * tracking is needed: each result row is self-identifying, and the - * beneficiary's tb_stoptb_diagnostics row (if any) is resolved live at - * import time. No vanID/servicePointID scoping either — same as export, - * MMU's per-van local database makes that implicit. + * Confirmed against a real results file (2026-08-18): the ID Generator app + * does NOT pass through any AMRIT-side beneficiary ID — the results CSV only + * has the original template columns (typeOfCaseFinding, caste, firstName, + * ..., hivStatus) plus its own generatedId/status/durationSec/error columns. + * There is deliberately no assumption that result-row order matches the + * uploaded file's row order either (unconfirmed, and a single + * reordered/dropped row would silently corrupt every row after it if wrong). + * + * Instead, each row is matched back to a beneficiary by content — + * normalized 10-digit primaryPhone plus a case-insensitive firstName match + * (see NikshayExportRepository.findMatchingBeneficiaryIds). This is a + * heuristic, not an exact key: anything other than exactly one match (zero, + * or more than one — e.g. a shared family phone number) is left for manual + * review rather than guessed. * * Row status handling: * - "success": generatedId is the new Nikshay ID — written as-is. @@ -61,7 +67,7 @@ @Service public class NikshayImportService { - private static final List REQUIRED_COLUMNS = List.of("benRegId", "firstName", "middleLastName", + private static final List REQUIRED_COLUMNS = List.of("primaryPhone", "firstName", "middleLastName", "generatedId", "status"); public record ImportRowResult(int rowIndex, Long benRegId, String firstName, String middleLastName, @@ -103,13 +109,24 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre String status = record.get("status").trim(); String generatedId = record.get("generatedId").trim(); - Long benRegId = parseBenRegId(record.get("benRegId")); - if (benRegId == null) { + String phoneDigits = normalizePhone(record.get("primaryPhone")); + if (phoneDigits == null) { failedRows.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, - "Row has a missing/invalid benRegId — was this file exported by AMRIT?")); + "Row has a missing/invalid primaryPhone — cannot match it back to a beneficiary.")); continue; } + List candidates = nikshayExportRepository.findMatchingBeneficiaryIds(phoneDigits, firstName); + if (candidates.size() != 1) { + String note = candidates.isEmpty() + ? "No beneficiary found matching this phone number and first name." + : "More than one beneficiary matches this phone number and first name (e.g. a shared " + + "family phone) — needs manual confirmation."; + needsReview.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, note)); + continue; + } + Long benRegId = candidates.get(0); + if ("success".equalsIgnoreCase(status) || "skipped".equalsIgnoreCase(status)) { String[] tokens = generatedId.isEmpty() ? new String[0] : generatedId.split("\\s+"); if (tokens.length == 1) { @@ -133,23 +150,26 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre failedRows); } - private static Long parseBenRegId(String raw) { - if (raw == null || raw.isBlank()) { + /** Same normalization as the export's validPhoneOrBlank, so a phone number + * round-trips consistently: strips non-digits, keeps the last 10, requires + * a valid-looking Indian mobile number. Null if it doesn't. */ + private static String normalizePhone(String raw) { + if (raw == null) { return null; } - try { - return Long.valueOf(raw.trim()); - } catch (NumberFormatException e) { - return null; + String digits = raw.replaceAll("[^0-9]", ""); + if (digits.length() > 10) { + digits = digits.substring(digits.length() - 10); } + return digits.matches("[1-9][0-9]{9}") ? digits : null; } private void writeNikshayId(LocalDate visitDate, Long benRegId, String nikshayId, String modifiedBy) { - Long diagnosticsId = nikshayExportRepository.findLatestDiagnosticsId(benRegId); - if (diagnosticsId != null) { - nikshayExportRepository.updateNikshayId(diagnosticsId, nikshayId, modifiedBy); + Long suspectedId = nikshayExportRepository.findLatestSuspectedId(benRegId); + if (suspectedId != null) { + nikshayExportRepository.updateNikshayId(suspectedId, nikshayId, modifiedBy); } else { - nikshayExportRepository.insertDiagnosticsWithNikshayId(benRegId, visitDate, nikshayId, modifiedBy); + nikshayExportRepository.insertSuspectedWithNikshayId(benRegId, visitDate, nikshayId, modifiedBy); } } } From e3f38d6b75723473f7b88ed59de4d1ec31b71d65 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 21:24:23 +0530 Subject: [PATCH 05/81] fix(stoptb): rewrite export query against verified real schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every table the export previously used for beneficiary/visit data turned out not to exist on a real server (i_beneficiary, I_bendemographics, tb_stoptb_visit) — confirmed via BadSqlGrammarException in production logs and direct schema inspection. Rewritten against tables and columns verified with real data: - Visit filter: t_benvisitdetail (VisitCategory = 'Stop TB'), not tb_stoptb_visit — confirmed 511 real rows vs. zero. - Beneficiary identity: MMU's datasource only connects to db_iemr, but beneficiary data lives in db_identity on the same physical server — reached via fully-qualified cross-schema table names: i_beneficiarymapping (BenRegId, unique) -> BenDetailsId -> i_beneficiarydetails (name/DOB/gender/caste/occupation/income/HIV, already denormalized, no extra master-table joins needed) and BenAddressId -> i_beneficiaryaddress (address/village/pincode), plus BenContactsId -> i_beneficiarycontacts (phone). - New confirmed real gap: many van-registered beneficiaries (i_beneficiarymapping.CreatedBy = 'reglocal') have a mapping row but their detail/address rows never synced to db_identity, sometimes for days. These are now silently skipped (no FirstName resolved), same category as an unresolved location — renamed countUnresolvedLocation -> countNotReadyToExport to reflect both cases. - Import's phone+name matching updated to the same verified join path. Tested against real data on a live server: 15 of 20 today's-date beneficiaries correctly skipped (identity never synced), 5 correctly resolved end-to-end including the full Nikshay location chain, producing valid CSV rows. Co-Authored-By: Claude Sonnet 5 --- .../stoptb/NikshayExportController.java | 6 +- .../repo/stoptb/NikshayExportRepository.java | 140 +++++++++++------- .../service/stoptb/NikshayExportService.java | 4 +- 3 files changed, 91 insertions(+), 59 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java index 7757528c..68b831f5 100644 --- a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -105,10 +105,10 @@ public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String } int excludedAlreadyGenerated; - int excludedUnresolvedLocation; + int excludedNotReadyToExport; try { excludedAlreadyGenerated = nikshayExportService.countAlreadyGenerated(fromDate, toDate); - excludedUnresolvedLocation = nikshayExportService.countUnresolvedLocation(fromDate, toDate); + excludedNotReadyToExport = nikshayExportService.countNotReadyToExport(fromDate, toDate); } catch (Exception e) { logger.error("Error preparing Nikshay beneficiary export", e); return ResponseEntity.status(500).body("Could not prepare the export"); @@ -129,7 +129,7 @@ public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String return ResponseEntity.ok().contentType(MediaType.parseMediaType("text/csv")) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"") .header("X-Excluded-Existing-Nikshay-Id-Count", String.valueOf(excludedAlreadyGenerated)) - .header("X-Excluded-Unresolved-Location-Count", String.valueOf(excludedUnresolvedLocation)) + .header("X-Excluded-Not-Ready-Count", String.valueOf(excludedNotReadyToExport)) .body(body); } diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index d6567cfe..737fb063 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -42,27 +42,54 @@ /** * Reads Stop TB beneficiaries for the Nikshay ID Generator CSV export. * - * MMU runs one local database per van, so a beneficiary visible here already - * belongs to the current van/service point by construction — there is - * deliberately no vanID/servicePointID filter, only a visit-date range. + * All table/column names here are verified against real data on a live + * server (2026-08-18), not assumed from convention — an earlier version of + * this class used i_beneficiary/I_bendemographics/tb_stoptb_visit, which + * turned out not to exist at all. The real picture: + * + * - Visit filter: db_iemr.t_benvisitdetail (VisitCategory = 'Stop TB'), + * NOT tb_stoptb_visit — confirmed 511 real rows on the reference server, + * vs. zero for the table this used to query. + * - Beneficiary identity: MMU's single datasource only connects to db_iemr, + * but the actual beneficiary/demographic tables live in db_identity, on + * the same physical MySQL server — reached here via fully-qualified + * cross-schema table names. The chain is + * db_identity.i_beneficiarymapping (BenRegId, unique) -> BenDetailsId -> + * db_identity.i_beneficiarydetails (name/DOB/gender/caste/occupation/ + * income/HIV — already denormalized to readable strings, no separate + * master-table joins needed) and BenAddressId -> + * db_identity.i_beneficiaryaddress (address/village/pincode), plus + * BenContactsId -> db_identity.i_beneficiarycontacts (phone). + * - Confirmed real gap, not a bug: many beneficiaries registered through + * the van-local registrar flow (i_beneficiarymapping.CreatedBy = + * 'reglocal') have a mapping row but their BenDetailsId/BenAddressId + * never synced to db_identity — sometimes for many days. Rows with no + * synced identity are silently left out (see streamPendingBeneficiaries) + * the same way an unresolved location is — there's nothing to put in a + * CSV row for someone whose name/address was never actually synced. * * Location columns (village/healthFacility/tu/district/state) are resolved * against Nikshay's own, isolated location hierarchy — m_nikshay_village → - * m_nikshay_facility → m_nikshay_tu → m_nikshay_district → m_nikshay_state — - * not AMRIT's standard masters. A Stop TB beneficiary's own - * I_bendemographics.DistrictBranchID holds their Nikshay Village ID (the same - * overload Common-API's NikshayAddressResolver relies on for the - * beneficiary-detail API), so every other location field is derived by - * walking up that one chain rather than trusting separately-stored - * block/district fields. + * m_nikshay_facility → m_nikshay_tu → m_nikshay_district → m_nikshay_state. + * This walk currently starts from the beneficiary's own personal + * i_beneficiaryaddress.CurrVillageId — confirmed on real data that this is + * AMRIT's own village ID, not Nikshay's (same numeric ID resolves to a + * different real place in each hierarchy), so this is a best-effort, + * unconfirmed mapping: it only produces a row when that AMRIT village ID + * happens to also be a valid Nikshay village ID, which will under-match. + * Whether Nikshay location should instead come from the camp/facility + * (via the logged-in worker's own assigned NikshayTUID/NikshayFacilityID + * on m_userservicerolemapping) rather than the beneficiary's personal + * address is still an open question — not yet resolved. * * Beneficiaries are silently left out of the streamed rows (see - * streamPendingBeneficiaries) in two cases — already having a Nikshay ID - * recorded, or their village not resolving all the way up to a state. Both - * are counted so callers can report totals, but there is deliberately no - * separate report of *who* was skipped or why. + * streamPendingBeneficiaries) in three cases — already having a Nikshay ID + * recorded, their identity never having synced to db_identity, or their + * village not resolving all the way up to a state. All three are counted so + * callers can report totals, but there is deliberately no separate report + * of *who* was skipped or why. * - * The Nikshay ID itself lives on tb_suspected.nikshay_id — not + * The Nikshay ID itself lives on db_iemr.tb_suspected.nikshay_id — not * tb_stoptb_diagnostics, which also has a nikshay_id column but is not the * table this feature writes to. */ @@ -89,49 +116,46 @@ public record NikshayRawRow(Long benRegId, String firstName, String middleLastNa // Placeholders in order: [1] fromDate (inclusive), [2] toDate-exclusive-upper-bound. private static final String BASE_SELECT = "SELECT " - + " b.BeneficiaryRegID AS benRegId, " - + " b.FirstName AS firstName, " - + " TRIM(CONCAT(COALESCE(b.MiddleName,''),' ',COALESCE(b.LastName,''))) AS middleLastName, " - + " TIMESTAMPDIFF(YEAR, b.DOB, CURDATE()) AS age, " - + " g.GenderName AS gender, " - + " (SELECT p.PhoneNo FROM i_benphonemap p WHERE p.BenificiaryRegID = b.BeneficiaryRegID " - + " AND p.Deleted = 0 ORDER BY p.BenPhMapID ASC LIMIT 1) AS phone, " - + " TRIM(CONCAT_WS(', ', d.AddressLine1, d.AddressLine2, d.AddressLine3, d.AddressLine4, d.AddressLine5)) AS address, " + + " m.BenRegId AS benRegId, " + + " d.FirstName AS firstName, " + + " TRIM(CONCAT(COALESCE(d.MiddleName,''),' ',COALESCE(d.LastName,''))) AS middleLastName, " + + " TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) AS age, " + + " d.Gender AS gender, " + + " c.PhoneNum1 AS phone, " + + " COALESCE(d.address, a.CurrAddressValue) AS address, " + " ns.StateName AS stateName, " + " nd.DistrictName AS districtName, " + " ntu.TUName AS tu, " + " nf.FacilityName AS healthFacility, " + " nv.VillageName AS village, " - + " d.PinCode AS pincode, " - + " ms.Status AS maritalStatus, " - + " c.CommunityType AS caste, " - + " occ.OccupationType AS occupation, " - + " inc.IncomeStatus AS socioeconomicStatus, " - + " (SELECT o.chief_complaint FROM tb_stoptb_general_opd o WHERE o.ben_reg_id = b.BeneficiaryRegID " + + " a.CurrPinCode AS pincode, " + + " d.MaritalStatus AS maritalStatus, " + + " d.community AS caste, " + + " d.occupation AS occupation, " + + " d.incomeStatus AS socioeconomicStatus, " + + " (SELECT o.chief_complaint FROM tb_stoptb_general_opd o WHERE o.ben_reg_id = m.BenRegId " + " AND o.deleted = 0 ORDER BY o.id DESC LIMIT 1) AS chiefComplaint, " - + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = b.BeneficiaryRegID " + + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = m.BenRegId " + " AND ge.deleted = 0 ORDER BY ge.id DESC LIMIT 1) AS hivStatus, " - + " b.IsHIVPos AS isHivPos, " - + " (SELECT s.nikshay_id FROM tb_suspected s WHERE s.benRegID = b.BeneficiaryRegID " + + " d.IsHIVPositive AS isHivPos, " + + " (SELECT s.nikshay_id FROM tb_suspected s WHERE s.benRegID = m.BenRegId " + " AND s.nikshay_id IS NOT NULL ORDER BY s.id DESC LIMIT 1) AS existingNikshayId " - + "FROM i_beneficiary b " - + "LEFT JOIN I_bendemographics d ON d.BeneficiaryRegID = b.BeneficiaryRegID " - + "LEFT JOIN m_gender g ON g.GenderID = b.GenderID " - + "LEFT JOIN m_maritalstatus ms ON ms.MaritalStatusID = b.MaritalStatusID " - + "LEFT JOIN m_community c ON c.CommunityID = d.CommunityID " - + "LEFT JOIN m_beneficiaryincomestatus inc ON inc.IncomeStatusID = d.IncomeStatusID " - + "LEFT JOIN m_beneficiaryoccupation occ ON occ.OccupationID = d.OccupationID " - // Stop TB's DistrictBranchID is overloaded to hold the Nikshay Village ID - // (see class Javadoc) — walk Nikshay's own hierarchy up from there. - + "LEFT JOIN m_nikshay_village nv ON nv.NikshayVillageID = d.DistrictBranchID AND nv.Deleted = 0 " + + "FROM db_identity.i_beneficiarymapping m " + + "LEFT JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = m.BenDetailsId AND d.Deleted = 0 " + + "LEFT JOIN db_identity.i_beneficiaryaddress a ON a.BenAddressID = m.BenAddressId " + + "LEFT JOIN db_identity.i_beneficiarycontacts c ON c.BenContactsId = m.BenContactsId " + // a.CurrVillageId is the beneficiary's own AMRIT village ID, not confirmed + // to be a Nikshay Village ID — see class Javadoc "still an open question". + + "LEFT JOIN m_nikshay_village nv ON nv.NikshayVillageID = a.CurrVillageId AND nv.Deleted = 0 " + "LEFT JOIN m_nikshay_facility nf ON nf.NikshayFacilityID = nv.NikshayFacilityID AND nf.Deleted = 0 " + "LEFT JOIN m_nikshay_tu ntu ON ntu.NikshayTUID = nf.NikshayTUID AND ntu.Deleted = 0 " + "LEFT JOIN m_nikshay_district nd ON nd.NikshayDistrictID = ntu.NikshayDistrictID AND nd.Deleted = 0 " + "LEFT JOIN m_nikshay_state ns ON ns.NikshayStateID = nd.NikshayStateID AND ns.Deleted = 0 " - + "WHERE b.Deleted = 0 " - + " AND b.BeneficiaryRegID IN ( " - + " SELECT DISTINCT v.beneficiary_reg_id FROM tb_stoptb_visit v " - + " WHERE v.visit_date >= ? AND v.visit_date < ? " + + "WHERE m.Deleted = 0 " + + " AND m.BenRegId IN ( " + + " SELECT DISTINCT v.BeneficiaryRegID FROM t_benvisitdetail v " + + " WHERE v.VisitCategory = 'Stop TB' AND v.Deleted = 0 " + + " AND v.VisitDateTime >= ? AND v.VisitDateTime < ? " + " )"; public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { @@ -140,9 +164,15 @@ public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { return count == null ? 0 : count; } - public int countUnresolvedLocation(LocalDate fromDate, LocalDate toDate) { + /** Counts beneficiaries skipped because their identity never synced to + * db_identity (no FirstName resolved at all) OR their location doesn't + * resolve through the Nikshay hierarchy — reported as one combined "not + * ready to export" count, since both are data-completeness gaps rather + * than a beneficiary genuinely not needing a Nikshay ID. */ + public int countNotReadyToExport(LocalDate fromDate, LocalDate toDate) { String sql = "SELECT COUNT(*) FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NULL " - + "AND (t.village IS NULL OR t.healthFacility IS NULL OR t.tu IS NULL " + + "AND (t.firstName IS NULL " + + "OR t.village IS NULL OR t.healthFacility IS NULL OR t.tu IS NULL " + "OR t.districtName IS NULL OR t.stateName IS NULL)"; Integer count = getJdbcTemplate().query(sql, pss(fromDate, toDate), rs -> rs.next() ? rs.getInt(1) : 0); return count == null ? 0 : count; @@ -151,10 +181,11 @@ public int countUnresolvedLocation(LocalDate fromDate, LocalDate toDate) { /** Streams every not-yet-Nikshay-ID'd beneficiary in the date range to * {@code rowConsumer} one row at a time, without materializing the full * result set in memory — safe for large date ranges. Silently skips any - * beneficiary whose Nikshay village doesn't resolve all the way up to a - * state (see class Javadoc). */ + * beneficiary whose identity never synced or whose Nikshay village + * doesn't resolve all the way up to a state (see class Javadoc). */ public void streamPendingBeneficiaries(LocalDate fromDate, LocalDate toDate, Consumer rowConsumer) { String sql = "SELECT * FROM (" + BASE_SELECT + ") t WHERE t.existingNikshayId IS NULL " + + "AND t.firstName IS NOT NULL " + "AND t.village IS NOT NULL AND t.healthFacility IS NOT NULL AND t.tu IS NOT NULL " + "AND t.districtName IS NOT NULL AND t.stateName IS NOT NULL"; JdbcTemplate jdbcTemplate = getJdbcTemplate(); @@ -202,10 +233,11 @@ private NikshayRawRow mapRow(ResultSet rs) throws SQLException { * callers must treat anything other than exactly one result as ambiguous * (e.g. a shared family phone number) rather than guessing. */ public List findMatchingBeneficiaryIds(String phoneDigits, String firstName) { - String sql = "SELECT DISTINCT b.BeneficiaryRegID FROM i_beneficiary b " - + "JOIN i_benphonemap p ON p.BenificiaryRegID = b.BeneficiaryRegID AND p.Deleted = 0 " - + "WHERE b.Deleted = 0 AND p.PhoneNo = ? AND LOWER(TRIM(b.FirstName)) = LOWER(TRIM(?))"; - return getJdbcTemplate().query(sql, (rs, rowNum) -> rs.getLong("BeneficiaryRegID"), phoneDigits, firstName); + String sql = "SELECT DISTINCT m.BenRegId FROM db_identity.i_beneficiarymapping m " + + "JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = m.BenDetailsId AND d.Deleted = 0 " + + "JOIN db_identity.i_beneficiarycontacts c ON c.BenContactsId = m.BenContactsId " + + "WHERE m.Deleted = 0 AND c.PhoneNum1 = ? AND LOWER(TRIM(d.FirstName)) = LOWER(TRIM(?))"; + return getJdbcTemplate().query(sql, (rs, rowNum) -> rs.getLong("BenRegId"), phoneDigits, firstName); } /** The most recent tb_suspected row for this beneficiary, if any — looked diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java index 44078859..adae98e4 100644 --- a/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayExportService.java @@ -113,8 +113,8 @@ public int countAlreadyGenerated(LocalDate fromDate, LocalDate toDate) { return nikshayExportRepository.countAlreadyGenerated(fromDate, toDate); } - public int countUnresolvedLocation(LocalDate fromDate, LocalDate toDate) { - return nikshayExportRepository.countUnresolvedLocation(fromDate, toDate); + public int countNotReadyToExport(LocalDate fromDate, LocalDate toDate) { + return nikshayExportRepository.countNotReadyToExport(fromDate, toDate); } /** Writes the CSV (header + one row per pending beneficiary) directly to From a73cb62bf117a480d90891b5720583bf6c7f3ff0 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 21:35:33 +0530 Subject: [PATCH 06/81] fix(stoptb): fix CSV streaming response (No converter for [...Lambda...]) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exportBeneficiariesCsv was declared ResponseEntity so it could return either a StreamingResponseBody (success) or a plain String (bad request/ error). Spring's StreamingResponseBodyReturnValueHandler decides whether to special-case the response based on the method's *declared* generic return type, not the runtime object — a wildcard resolves to Object, StreamingResponseBody isn't assignable from Object, so it falls through to normal HttpMessageConverters, which can't serialize a raw lambda: "No converter for [...NikshayExportController$$Lambda...] with preset Content-Type 'text/csv'" (confirmed in production logs, 2026-08-18). Fixed by keeping the return type consistently ResponseEntity and wrapping error messages as a StreamingResponseBody that writes the message as bytes, instead of returning a plain String on any path. Co-Authored-By: Claude Sonnet 5 --- .../stoptb/NikshayExportController.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java index 68b831f5..960d9ac4 100644 --- a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -89,7 +89,7 @@ private String currentUsername(HttpServletRequest request) { @Operation(summary = "Download Stop TB beneficiaries for a date range as a CSV formatted for the Nikshay ID Generator") @GetMapping(value = "/exportBeneficiariesCsv") - public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String fromDateStr, + public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String fromDateStr, @RequestParam("toDate") String toDateStr) { LocalDate fromDate; @@ -98,10 +98,10 @@ public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String fromDate = LocalDate.parse(fromDateStr, DATE_FMT); toDate = LocalDate.parse(toDateStr, DATE_FMT); } catch (DateTimeParseException e) { - return ResponseEntity.badRequest().body("fromDate/toDate must be in YYYY-MM-DD format"); + return ResponseEntity.badRequest().body(errorBody("fromDate/toDate must be in YYYY-MM-DD format")); } if (toDate.isBefore(fromDate)) { - return ResponseEntity.badRequest().body("toDate must be on or after fromDate"); + return ResponseEntity.badRequest().body(errorBody("toDate must be on or after fromDate")); } int excludedAlreadyGenerated; @@ -111,7 +111,7 @@ public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String excludedNotReadyToExport = nikshayExportService.countNotReadyToExport(fromDate, toDate); } catch (Exception e) { logger.error("Error preparing Nikshay beneficiary export", e); - return ResponseEntity.status(500).body("Could not prepare the export"); + return ResponseEntity.status(500).body(errorBody("Could not prepare the export")); } StreamingResponseBody body = outputStream -> { @@ -133,6 +133,19 @@ public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String .body(body); } + /** Wraps a plain error message as a StreamingResponseBody so every return + * path of exportBeneficiariesCsv shares one concrete generic type + * (ResponseEntity<StreamingResponseBody>). A wildcard/Object-typed + * ResponseEntity here breaks Spring's StreamingResponseBodyReturnValueHandler + * — it decides whether to stream based on the method's *declared* generic + * return type, not the runtime object, so a wildcard falls back to normal + * HttpMessageConverters, which can't write a raw StreamingResponseBody + * lambda and throw "No converter for [...Lambda...]" (confirmed in + * production logs, 2026-08-18). */ + private StreamingResponseBody errorBody(String message) { + return outputStream -> outputStream.write(message.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + } + @Operation(summary = "Upload the Nikshay ID Generator app's results CSV to write generated Nikshay IDs " + "back onto the beneficiaries — each row is matched by its own benRegId column, " + "a pass-through field the export added that the ID Generator app never touches") From 605ce789853fdfea1f9211dc6c08b48e5d51823e Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 21:43:54 +0530 Subject: [PATCH 07/81] fix(stoptb): allow REGISTRAR role on Nikshay export/import endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @PreAuthorize only listed clinical roles (Nurse/Doctor/Pharmacist/Lab Technician/etc.), copied from sn/nikshaya's original controller without verifying against real Stop TB camp roles. Confirmed in production: AccessDeniedException for a real Stop TB user (stoptb1) whose roles (checked directly via v_userservicerolemapping) include Registrar, Nurse, Doctor, Pharmacist, Lab Technician, Data Sync — Registrar was the one missing from the allow-list. Downloading/uploading a beneficiary CSV is fundamentally a registrar task, so this was very likely the actual active session role denied by AccessDeniedException in the logs. Co-Authored-By: Claude Sonnet 5 --- .../com/iemr/mmu/controller/stoptb/NikshayExportController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java index 960d9ac4..660b15ca 100644 --- a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -57,7 +57,7 @@ */ @RestController @RequestMapping(value = "/stopTb/nikshay", headers = "Authorization") -@PreAuthorize("hasRole('NURSE') || hasRole('PHARMACIST') || hasRole('LABTECHNICIAN') || hasRole('DOCTOR') || hasRole('LAB_TECHNICIAN') || hasRole('TC_SPECIALIST') || hasRole('ONCOLOGIST') || hasRole('RADIOLOGIST')") +@PreAuthorize("hasRole('REGISTRAR') || hasRole('NURSE') || hasRole('PHARMACIST') || hasRole('LABTECHNICIAN') || hasRole('DOCTOR') || hasRole('LAB_TECHNICIAN') || hasRole('TC_SPECIALIST') || hasRole('ONCOLOGIST') || hasRole('RADIOLOGIST')") public class NikshayExportController { private static final Logger logger = LoggerFactory.getLogger(NikshayExportController.class); private static final DateTimeFormatter DATE_FMT = DateTimeFormatter.ISO_LOCAL_DATE; From 3389f68ed7dfc22546b68e76be4cc91dbe3372e0 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 22:03:27 +0530 Subject: [PATCH 08/81] fix(stoptb): remove @PreAuthorize role gate on Nikshay export/import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Role-based gating proved fragile — wrong role list on the first attempt, still AccessDeniedException after adding REGISTRAR. SecurityConfig's anyRequest().authenticated() still requires a logged-in user; dropping the role-specific check per explicit request rather than guessing a third time. Co-Authored-By: Claude Sonnet 5 --- .../mmu/controller/stoptb/NikshayExportController.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java index 660b15ca..4404838d 100644 --- a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -31,7 +31,6 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; -import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestMapping; @@ -57,7 +56,11 @@ */ @RestController @RequestMapping(value = "/stopTb/nikshay", headers = "Authorization") -@PreAuthorize("hasRole('REGISTRAR') || hasRole('NURSE') || hasRole('PHARMACIST') || hasRole('LABTECHNICIAN') || hasRole('DOCTOR') || hasRole('LAB_TECHNICIAN') || hasRole('TC_SPECIALIST') || hasRole('ONCOLOGIST') || hasRole('RADIOLOGIST')") +// No @PreAuthorize role gate — any authenticated user can hit these endpoints +// (SecurityConfig's anyRequest().authenticated() still applies). Role-based +// gating here proved fragile: it took two attempts to get the role list +// right, and it was never actually confirmed against the deployed build +// since fixes here weren't pushed before being tested (2026-08-18). public class NikshayExportController { private static final Logger logger = LoggerFactory.getLogger(NikshayExportController.class); private static final DateTimeFormatter DATE_FMT = DateTimeFormatter.ISO_LOCAL_DATE; From 5c30cd8ae47dceb9816bc68fcceb2a1c3984b249 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Tue, 18 Aug 2026 22:17:09 +0530 Subject: [PATCH 09/81] fix(stoptb): write CSV synchronously instead of via StreamingResponseBody MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the persistent AccessDeniedException (survived removing @PreAuthorize entirely): StreamingResponseBody makes Spring process the response body on a Servlet async re-dispatch. Confirmed from production logs (2026-08-18) — two AccessDeniedExceptions per request, the second via ApplicationDispatcher/AsyncContextImpl. Spring Security's own core filter chain is registered for ASYNC dispatch by default and re-runs, but this app's custom JwtUserIdValidationFilter/RoleAuthenticationFilter (which read the JWT cookie and establish who's logged in) are not — AnonymousAuthenticationFilter fires instead on the second pass, and the global anyRequest().authenticated() rule denies it. Fixed by writing the CSV directly and synchronously onto HttpServletResponse instead of returning a StreamingResponseBody — avoids async dispatch entirely, so there's no second security pass to fail. Co-Authored-By: Claude Sonnet 5 --- .../stoptb/NikshayExportController.java | 66 +++++++++---------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java index 4404838d..5ae19995 100644 --- a/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java +++ b/src/main/java/com/iemr/mmu/controller/stoptb/NikshayExportController.java @@ -37,7 +37,6 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; import com.iemr.mmu.service.stoptb.NikshayExportService; import com.iemr.mmu.service.stoptb.NikshayImportService; @@ -46,6 +45,7 @@ import io.swagger.v3.oas.annotations.Operation; import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; /** * Downloads/uploads the Stop TB Nikshay ID Generator CSV. @@ -90,10 +90,21 @@ private String currentUsername(HttpServletRequest request) { } } + /** Writes the CSV directly and synchronously onto the servlet response, + * instead of returning a StreamingResponseBody. StreamingResponseBody + * makes Spring process the body on a Servlet-async re-dispatch — Spring + * Security's filter chain re-runs on that async dispatch, and the + * SecurityContext doesn't reliably carry over to it, so the global + * anyRequest().authenticated() rule was denying the *second* pass even + * though the initial request authenticated fine (confirmed in production + * logs, 2026-08-18: two AccessDeniedExceptions for the same request, the + * second one through ApplicationDispatcher/AsyncContextImpl). Writing + * synchronously avoids async dispatch entirely, so there's no second + * security pass to fail. */ @Operation(summary = "Download Stop TB beneficiaries for a date range as a CSV formatted for the Nikshay ID Generator") @GetMapping(value = "/exportBeneficiariesCsv") - public ResponseEntity exportBeneficiariesCsv(@RequestParam("fromDate") String fromDateStr, - @RequestParam("toDate") String toDateStr) { + public void exportBeneficiariesCsv(@RequestParam("fromDate") String fromDateStr, + @RequestParam("toDate") String toDateStr, HttpServletResponse response) throws java.io.IOException { LocalDate fromDate; LocalDate toDate; @@ -101,10 +112,12 @@ public ResponseEntity exportBeneficiariesCsv(@RequestPara fromDate = LocalDate.parse(fromDateStr, DATE_FMT); toDate = LocalDate.parse(toDateStr, DATE_FMT); } catch (DateTimeParseException e) { - return ResponseEntity.badRequest().body(errorBody("fromDate/toDate must be in YYYY-MM-DD format")); + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "fromDate/toDate must be in YYYY-MM-DD format"); + return; } if (toDate.isBefore(fromDate)) { - return ResponseEntity.badRequest().body(errorBody("toDate must be on or after fromDate")); + response.sendError(HttpServletResponse.SC_BAD_REQUEST, "toDate must be on or after fromDate"); + return; } int excludedAlreadyGenerated; @@ -114,39 +127,24 @@ public ResponseEntity exportBeneficiariesCsv(@RequestPara excludedNotReadyToExport = nikshayExportService.countNotReadyToExport(fromDate, toDate); } catch (Exception e) { logger.error("Error preparing Nikshay beneficiary export", e); - return ResponseEntity.status(500).body(errorBody("Could not prepare the export")); + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Could not prepare the export"); + return; } - StreamingResponseBody body = outputStream -> { - try { - nikshayExportService.streamBeneficiariesCsv(fromDate, toDate, outputStream); - } catch (Exception e) { - // The HTTP status/headers are already committed by the time streaming - // starts, so a mid-stream failure can only be logged, not surfaced - // as a clean error response. - logger.error("Error streaming Nikshay beneficiary CSV", e); - } - }; - String filename = "nikshay-beneficiaries-" + fromDate + "-to-" + toDate + ".csv"; - return ResponseEntity.ok().contentType(MediaType.parseMediaType("text/csv")) - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"") - .header("X-Excluded-Existing-Nikshay-Id-Count", String.valueOf(excludedAlreadyGenerated)) - .header("X-Excluded-Not-Ready-Count", String.valueOf(excludedNotReadyToExport)) - .body(body); - } + response.setContentType("text/csv"); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\""); + response.setHeader("X-Excluded-Existing-Nikshay-Id-Count", String.valueOf(excludedAlreadyGenerated)); + response.setHeader("X-Excluded-Not-Ready-Count", String.valueOf(excludedNotReadyToExport)); - /** Wraps a plain error message as a StreamingResponseBody so every return - * path of exportBeneficiariesCsv shares one concrete generic type - * (ResponseEntity<StreamingResponseBody>). A wildcard/Object-typed - * ResponseEntity here breaks Spring's StreamingResponseBodyReturnValueHandler - * — it decides whether to stream based on the method's *declared* generic - * return type, not the runtime object, so a wildcard falls back to normal - * HttpMessageConverters, which can't write a raw StreamingResponseBody - * lambda and throw "No converter for [...Lambda...]" (confirmed in - * production logs, 2026-08-18). */ - private StreamingResponseBody errorBody(String message) { - return outputStream -> outputStream.write(message.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + try { + nikshayExportService.streamBeneficiariesCsv(fromDate, toDate, response.getOutputStream()); + } catch (Exception e) { + // Headers are already committed by the time streaming starts, so a + // mid-stream failure can only be logged, not surfaced as a clean + // error response. + logger.error("Error streaming Nikshay beneficiary CSV", e); + } } @Operation(summary = "Upload the Nikshay ID Generator app's results CSV to write generated Nikshay IDs " From 79d8422d2cccfb040446b4326a921815373a9d9c Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Wed, 19 Aug 2026 09:54:56 +0530 Subject: [PATCH 10/81] fix(stoptb): read phone/address from the columns actually populated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified against real data: PhoneNum1 and CurrAddressValue are empty on every real beneficiary checked — the actual data lives in PreferredPhoneNum (phone) and CurrAddrLine1/2/3 + CurrHabitation (address), confirmed by direct query on real records. Falls back through PhoneNum1/PhoneNum2 if PreferredPhoneNum is empty. Address line parts are individually NULLIF'd before CONCAT_WS, since blank (not NULL) line fields otherwise leave stray ", ," artifacts. Co-Authored-By: Claude Sonnet 5 --- .../mmu/repo/stoptb/NikshayExportRepository.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 737fb063..322fc3a6 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -121,8 +121,16 @@ public record NikshayRawRow(Long benRegId, String firstName, String middleLastNa + " TRIM(CONCAT(COALESCE(d.MiddleName,''),' ',COALESCE(d.LastName,''))) AS middleLastName, " + " TIMESTAMPDIFF(YEAR, d.DOB, CURDATE()) AS age, " + " d.Gender AS gender, " - + " c.PhoneNum1 AS phone, " - + " COALESCE(d.address, a.CurrAddressValue) AS address, " + // PhoneNum1 is usually empty on real data — PreferredPhoneNum is the + // one actually populated at registration; fall back through the rest. + + " COALESCE(NULLIF(c.PreferredPhoneNum,''), NULLIF(c.PhoneNum1,''), NULLIF(c.PhoneNum2,'')) AS phone, " + // CurrAddressValue is usually empty on real data too — build from the + // actual line fields instead, same as CurrAddrLine1 etc. being populated. + + " COALESCE(NULLIF(d.address,''), NULLIF(a.CurrAddressValue,''), " + // CONCAT_WS skips NULLs but not empty strings, so each part needs its + // own NULLIF first or blank line fields leave stray ", ," artifacts. + + " NULLIF(TRIM(CONCAT_WS(', ', NULLIF(a.CurrAddrLine1,''), NULLIF(a.CurrAddrLine2,''), " + + " NULLIF(a.CurrAddrLine3,''), NULLIF(a.CurrHabitation,''))),'')) AS address, " + " ns.StateName AS stateName, " + " nd.DistrictName AS districtName, " + " ntu.TUName AS tu, " From c3faa90c3bc514164bb907b4b7ee418a67006c42 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Wed, 19 Aug 2026 12:46:07 +0530 Subject: [PATCH 11/81] fix(stoptb): import phone matching missed PreferredPhoneNum too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit findMatchingBeneficiaryIds only checked c.PhoneNum1 — same bug already fixed on the export side (commit 79d8422d) but missed here. Confirmed in production: two real beneficiaries (17877 'K', 17875 'U') with PhoneNum1 IS NULL but PreferredPhoneNum populated ('9999999999') both came back "no beneficiary found" on upload. Now checks PreferredPhoneNum/PhoneNum1/PhoneNum2, same as the export's column preference order. Verified against real data: 17877 now matches correctly for phone '9999999999' + firstName 'K'. Co-Authored-By: Claude Sonnet 5 --- .../com/iemr/mmu/repo/stoptb/NikshayExportRepository.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 322fc3a6..28268b3d 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -241,10 +241,16 @@ private NikshayRawRow mapRow(ResultSet rs) throws SQLException { * callers must treat anything other than exactly one result as ambiguous * (e.g. a shared family phone number) rather than guessing. */ public List findMatchingBeneficiaryIds(String phoneDigits, String firstName) { + // Same column-order fix as the export's phone selection: PhoneNum1 is + // usually empty on real data, PreferredPhoneNum is what's actually + // populated (confirmed: two real beneficiaries with PhoneNum1 IS NULL + // but PreferredPhoneNum populated failed to match here before this fix). String sql = "SELECT DISTINCT m.BenRegId FROM db_identity.i_beneficiarymapping m " + "JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = m.BenDetailsId AND d.Deleted = 0 " + "JOIN db_identity.i_beneficiarycontacts c ON c.BenContactsId = m.BenContactsId " - + "WHERE m.Deleted = 0 AND c.PhoneNum1 = ? AND LOWER(TRIM(d.FirstName)) = LOWER(TRIM(?))"; + + "WHERE m.Deleted = 0 " + + " AND ? IN (c.PreferredPhoneNum, c.PhoneNum1, c.PhoneNum2) " + + " AND LOWER(TRIM(d.FirstName)) = LOWER(TRIM(?))"; return getJdbcTemplate().query(sql, (rs, rowNum) -> rs.getLong("BenRegId"), phoneDigits, firstName); } From 71ddd404c9421272e4009395876a4cdb90a3d706 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Wed, 19 Aug 2026 13:15:25 +0530 Subject: [PATCH 12/81] feat(stoptb): use benRegId directly when present, fall back to phone+name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a results row carries a parseable benRegId (e.g. a manually-rebuilt test file, or a future ID Generator version that keeps it), use it directly — exact match, no ambiguity possible, skips the phone+name lookup entirely. Falls back to the existing phone+name matching only when benRegId is absent or unparseable, which is what a genuine Nikshay portal results file looks like (confirmed no ID column at all). Makes test files with benRegId (e.g. a rebuilt export) match instantly and unambiguously, while real portal files keep working exactly as before via the phone+name path. Co-Authored-By: Claude Sonnet 5 --- .../service/stoptb/NikshayImportService.java | 64 +++++++++++++------ 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java index 36033dc6..ebc81e4e 100644 --- a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -49,12 +49,15 @@ * uploaded file's row order either (unconfirmed, and a single * reordered/dropped row would silently corrupt every row after it if wrong). * - * Instead, each row is matched back to a beneficiary by content — - * normalized 10-digit primaryPhone plus a case-insensitive firstName match - * (see NikshayExportRepository.findMatchingBeneficiaryIds). This is a - * heuristic, not an exact key: anything other than exactly one match (zero, - * or more than one — e.g. a shared family phone number) is left for manual - * review rather than guessed. + * If a row happens to carry a benRegId column with a parseable value (e.g. a + * manually-rebuilt test file, or some future ID Generator version that keeps + * it) it's used directly — exact, no ambiguity possible. Otherwise, each row + * is matched back to a beneficiary by content — normalized 10-digit + * primaryPhone plus a case-insensitive firstName match (see + * NikshayExportRepository.findMatchingBeneficiaryIds). This is a heuristic, + * not an exact key: anything other than exactly one match (zero, or more + * than one — e.g. a shared family phone number) is left for manual review + * rather than guessed. * * Row status handling: * - "success": generatedId is the new Nikshay ID — written as-is. @@ -85,6 +88,7 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre throws Exception { List records; boolean hasErrorColumn; + boolean hasBenRegIdColumn; CSVFormat format = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).setTrim(true).build(); try (CSVParser parser = new CSVParser(new InputStreamReader(csvInputStream, StandardCharsets.UTF_8), format)) { @@ -95,6 +99,7 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre } } hasErrorColumn = header.containsKey("error"); + hasBenRegIdColumn = header.containsKey("benRegId"); records = parser.getRecords(); } @@ -109,23 +114,29 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre String status = record.get("status").trim(); String generatedId = record.get("generatedId").trim(); - String phoneDigits = normalizePhone(record.get("primaryPhone")); - if (phoneDigits == null) { - failedRows.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, - "Row has a missing/invalid primaryPhone — cannot match it back to a beneficiary.")); - continue; - } + Long benRegId = hasBenRegIdColumn ? parseBenRegId(record.get("benRegId")) : null; + if (benRegId == null) { + // No usable benRegId on this row — fall back to phone+name matching, + // the only option against a genuine Nikshay portal results file. + String phoneDigits = normalizePhone(record.get("primaryPhone")); + if (phoneDigits == null) { + failedRows.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, + "Row has a missing/invalid primaryPhone — cannot match it back to a beneficiary.")); + continue; + } - List candidates = nikshayExportRepository.findMatchingBeneficiaryIds(phoneDigits, firstName); - if (candidates.size() != 1) { - String note = candidates.isEmpty() - ? "No beneficiary found matching this phone number and first name." - : "More than one beneficiary matches this phone number and first name (e.g. a shared " - + "family phone) — needs manual confirmation."; - needsReview.add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, note)); - continue; + List candidates = nikshayExportRepository.findMatchingBeneficiaryIds(phoneDigits, firstName); + if (candidates.size() != 1) { + String note = candidates.isEmpty() + ? "No beneficiary found matching this phone number and first name." + : "More than one beneficiary matches this phone number and first name (e.g. a shared " + + "family phone) — needs manual confirmation."; + needsReview + .add(new ImportRowResult(i, null, firstName, middleLastName, status, generatedId, note)); + continue; + } + benRegId = candidates.get(0); } - Long benRegId = candidates.get(0); if ("success".equalsIgnoreCase(status) || "skipped".equalsIgnoreCase(status)) { String[] tokens = generatedId.isEmpty() ? new String[0] : generatedId.split("\\s+"); @@ -150,6 +161,17 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre failedRows); } + private static Long parseBenRegId(String raw) { + if (raw == null || raw.isBlank()) { + return null; + } + try { + return Long.valueOf(raw.trim()); + } catch (NumberFormatException e) { + return null; + } + } + /** Same normalization as the export's validPhoneOrBlank, so a phone number * round-trips consistently: strips non-digits, keeps the last 10, requires * a valid-looking Indian mobile number. Null if it doesn't. */ From 8597bbad10d7c5a43886981c0ac66636d25bdf13 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 20 Aug 2026 16:00:41 +0530 Subject: [PATCH 13/81] fix: add the endpoints and schema for downsync from central to local --- src/main/environment/common_ci.properties | 9 + src/main/environment/common_docker.properties | 181 +++++++++-------- .../environment/common_example.properties | 9 + .../DownSyncTableDetail.java | 185 ++++++++++++++++++ .../DownSyncTableDetailRepo.java | 41 ++++ 5 files changed, 339 insertions(+), 86 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java create mode 100644 src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java diff --git a/src/main/environment/common_ci.properties b/src/main/environment/common_ci.properties index 918156fa..faf5180f 100644 --- a/src/main/environment/common_ci.properties +++ b/src/main/environment/common_ci.properties @@ -51,6 +51,15 @@ dataSyncTransactionDownloadUrl=@env.MMU_CENTRAL_SERVER@dataSync/server-to-van-tr ## inventory transaction processed flag update URL, central, mmu dataSyncProcessedFlagUpdate=@env.MMU_CENTRAL_SERVER@dataSync/updateProcessedFlagPostDownload +## down-sync (central -> local) data API, central, mmu +downSyncDataUrl=@env.MMU_CENTRAL_SERVER@dataSync/server-to-van-downsync + +## down-sync (central -> local) flag update API, central, mmu +downSyncFlagUpdateUrl=@env.MMU_CENTRAL_SERVER@dataSync/updateDownSyncFlagPostDownload + +## mark an up-synced record as already delivered to the van (central only) +downSync.markDeliveredOnUpSync=true + fileBasePath =@env.MMU_FILE_BASE_PATH@ diff --git a/src/main/environment/common_docker.properties b/src/main/environment/common_docker.properties index 52298c5f..28f6b89e 100644 --- a/src/main/environment/common_docker.properties +++ b/src/main/environment/common_docker.properties @@ -1,87 +1,96 @@ -# local env -# DB Connections -spring.datasource.url=${DATABASE_URL} -spring.datasource.username=${DATABASE_USERNAME} -spring.datasource.password=${DATABASE_PASSWORD} -spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver - -## Carestream URLs, local -carestreamOrderCreateURL=${COMMON_API}carestream/createOrder - -## Identity - Common URLs -# Registration -registrationUrl =${COMMON_API}beneficiary/createBeneficiary - -registrarQuickSearchByIdUrl =${COMMON_API}beneficiary/searchUserByID - -registrarQuickSearchByPhoneNoUrl =${COMMON_API}beneficiary/searchUserByPhone - -getBenImageFromIdentity =${IDENTITY_API_BASE_URL}id/benImageByBenRegID - -beneficiaryEditUrl =${COMMON_API}beneficiary/updateBenefciaryDetails - -registrarAdvanceSearchUrl =${COMMON_API}beneficiary/searchBeneficiary - -## Data Sync API, central -dataSyncUploadUrl=${MMU_CENTRAL_SERVER}dataSync/van-to-server - -## Data download API, central -dataSyncDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van - -## TC specialist slot booking, # not required -tcSpecialistSlotBook=${TM_API}schedule/bookSlot - -## TC specialist slot cancelling, # not required -tcSpecialistSlotCancel=${TM_API}schedule/cancelBookedSlot - -## benGenUrl, common, central -benGenUrlCentral=${COMMON_API_OPEN_SYNC}beneficiary/generateBeneficiaryIDs - -## benImportUrl, identity, local -benImportUrlLocal=${IDENTITY_API_BASE_URL}id/saveGeneratedBenIDToLocalServer -mmucentralserver = ${MMU_CENTRAL_SERVER}common/get/Case-sheet/centralServerTMCaseSheet - -tmCentralServer = ${TM_CENTRAL_SERVER}common/get/Case-sheet/printData - -specialistSign=${COMMON_API}signature1/getSignClass - -## inventory transaction download URL, central, mmu -dataSyncTransactionDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van-transactional - -## inventory transaction processed flag update URL, central, mmu -dataSyncProcessedFlagUpdate=${MMU_CENTRAL_SERVER}dataSync/updateProcessedFlagPostDownload - - -fileBasePath =${MMU_FILE_BASE_PATH} - -## data sync batch size -BATCH_SIZE=30 -snomedCTPageSize=10 -nurseWL=7 -docWL=7 -pharmaWL=7 -labWL=7 -radioWL=7 -oncoWL=7 - -## For File Sync -serverIP=${FILE_SYNC_SERVER_IP} -serverDomain=${FILE_SYNC_SERVER_DOMAIN} -serverUserName=${FILE_SYNC_SERVER_USERNAME} -serverPassword=${FILE_SYNC_SERVER_PASSWORD} -localFolderToSync=${LOCAL_FOLDER_TO_SYNC} -serverFolder=MMUSync -getServerCredentialURL=${MMU_API}fileSyncController/getServerCredential -tmReferCheckValue=TM -### Redis IP -spring.redis.host=${REDIS_HOST} - -jwt.secret=${JWT_SECRET_KEY} - - -#ELK logging file name -logging.path=logs -logging.file.name=${MMU_API_LOGGING_FILE_NAME} - -springdoc.api-docs.enabled=${SWAGGER_DOC_ENABLED} +# local env +# DB Connections +spring.datasource.url=${DATABASE_URL} +spring.datasource.username=${DATABASE_USERNAME} +spring.datasource.password=${DATABASE_PASSWORD} +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver + +## Carestream URLs, local +carestreamOrderCreateURL=${COMMON_API}carestream/createOrder + +## Identity - Common URLs +# Registration +registrationUrl =${COMMON_API}beneficiary/createBeneficiary + +registrarQuickSearchByIdUrl =${COMMON_API}beneficiary/searchUserByID + +registrarQuickSearchByPhoneNoUrl =${COMMON_API}beneficiary/searchUserByPhone + +getBenImageFromIdentity =${IDENTITY_API_BASE_URL}id/benImageByBenRegID + +beneficiaryEditUrl =${COMMON_API}beneficiary/updateBenefciaryDetails + +registrarAdvanceSearchUrl =${COMMON_API}beneficiary/searchBeneficiary + +## Data Sync API, central +dataSyncUploadUrl=${MMU_CENTRAL_SERVER}dataSync/van-to-server + +## Data download API, central +dataSyncDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van + +## TC specialist slot booking, # not required +tcSpecialistSlotBook=${TM_API}schedule/bookSlot + +## TC specialist slot cancelling, # not required +tcSpecialistSlotCancel=${TM_API}schedule/cancelBookedSlot + +## benGenUrl, common, central +benGenUrlCentral=${COMMON_API_OPEN_SYNC}beneficiary/generateBeneficiaryIDs + +## benImportUrl, identity, local +benImportUrlLocal=${IDENTITY_API_BASE_URL}id/saveGeneratedBenIDToLocalServer +mmucentralserver = ${MMU_CENTRAL_SERVER}common/get/Case-sheet/centralServerTMCaseSheet + +tmCentralServer = ${TM_CENTRAL_SERVER}common/get/Case-sheet/printData + +specialistSign=${COMMON_API}signature1/getSignClass + +## inventory transaction download URL, central, mmu +dataSyncTransactionDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van-transactional + +## inventory transaction processed flag update URL, central, mmu +dataSyncProcessedFlagUpdate=${MMU_CENTRAL_SERVER}dataSync/updateProcessedFlagPostDownload + +## down-sync (central -> local) data API, central, mmu +downSyncDataUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van-downsync + +## down-sync (central -> local) flag update API, central, mmu +downSyncFlagUpdateUrl=${MMU_CENTRAL_SERVER}dataSync/updateDownSyncFlagPostDownload + +## mark an up-synced record as already delivered to the van (central only) +downSync.markDeliveredOnUpSync=true + + +fileBasePath =${MMU_FILE_BASE_PATH} + +## data sync batch size +BATCH_SIZE=30 +snomedCTPageSize=10 +nurseWL=7 +docWL=7 +pharmaWL=7 +labWL=7 +radioWL=7 +oncoWL=7 + +## For File Sync +serverIP=${FILE_SYNC_SERVER_IP} +serverDomain=${FILE_SYNC_SERVER_DOMAIN} +serverUserName=${FILE_SYNC_SERVER_USERNAME} +serverPassword=${FILE_SYNC_SERVER_PASSWORD} +localFolderToSync=${LOCAL_FOLDER_TO_SYNC} +serverFolder=MMUSync +getServerCredentialURL=${MMU_API}fileSyncController/getServerCredential +tmReferCheckValue=TM +### Redis IP +spring.redis.host=${REDIS_HOST} + +jwt.secret=${JWT_SECRET_KEY} + + +#ELK logging file name +logging.path=logs +logging.file.name=${MMU_API_LOGGING_FILE_NAME} + +springdoc.api-docs.enabled=${SWAGGER_DOC_ENABLED} springdoc.swagger-ui.enabled=${SWAGGER_DOC_ENABLED} \ No newline at end of file diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 7157c564..294c0e45 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -64,6 +64,15 @@ dataSyncTransactionDownloadUrl=http://10.208.122.38:8080/mmu-api/dataSync/server ## inventory transaction processed flag update URL, central, mmu dataSyncProcessedFlagUpdate=http://10.208.122.38:8080/mmu-api/dataSync/updateProcessedFlagPostDownload +## down-sync (central -> local) data API, central, mmu +downSyncDataUrl=http://10.208.122.38:8080/mmu-api/dataSync/server-to-van-downsync + +## down-sync (central -> local) flag update API, central, mmu +downSyncFlagUpdateUrl=http://10.208.122.38:8080/mmu-api/dataSync/updateDownSyncFlagPostDownload + +## mark an up-synced record as already delivered to the van (central only) +downSync.markDeliveredOnUpSync=true + ## fetching the TM case sheet for MMU. mmucentralserver = /mmu-api/common/get/Case-sheet/centralServerTMCaseSheet diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java new file mode 100644 index 00000000..c2f18050 --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -0,0 +1,185 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import com.google.gson.annotations.Expose; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.Transient; + +/*** + * Down-sync table configuration (central -> local). Analogous to + * {@link SyncUtilityClass} (m_synctabledetail) which drives the up-sync. + */ +@Entity +@Table(name = "m_downsynctabledetail") +public class DownSyncTableDetail { + + public static final String TABLE_TYPE_MASTER = "MASTER"; + public static final String TABLE_TYPE_TRANSACTIONAL = "TRANSACTIONAL"; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + @Expose + @Column(name = "DownSyncTableDetailID", updatable = false) + private Integer downSyncTableDetailID; + @Expose + @Column(name = "SchemaName") + private String schemaName; + @Expose + @Column(name = "TableName") + private String tableName; + /** columns to SELECT from central */ + @Expose + @Column(name = "ServerColumnName") + private String serverColumnName; + /** columns in local, positionally mapped with {@link #serverColumnName} */ + @Expose + @Column(name = "VanColumnName") + private String vanColumnName; + /** local auto-increment PK column - skipped on INSERT */ + @Expose + @Column(name = "VanAutoIncColumnName") + private String vanAutoIncColumnName; + /** MASTER = full pull, no VanID filter / TRANSACTIONAL = VanID + DownSynced filter */ + @Expose + @Column(name = "TableType") + private String tableType; + /** lower number syncs first - enforces the FK dependency chain */ + @Expose + @Column(name = "SyncOrder") + private Integer syncOrder; + @Expose + @Column(name = "IsActive") + private Boolean isActive; + + @Transient + @Expose + private Integer vanID; + @Transient + @Expose + private Integer providerServiceMapID; + + public DownSyncTableDetail() { + } + + public boolean isMasterTable() { + return tableType == null || TABLE_TYPE_MASTER.equalsIgnoreCase(tableType); + } + + public boolean isTransactionalTable() { + return TABLE_TYPE_TRANSACTIONAL.equalsIgnoreCase(tableType); + } + + public Integer getDownSyncTableDetailID() { + return downSyncTableDetailID; + } + + public void setDownSyncTableDetailID(Integer downSyncTableDetailID) { + this.downSyncTableDetailID = downSyncTableDetailID; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getServerColumnName() { + return serverColumnName; + } + + public void setServerColumnName(String serverColumnName) { + this.serverColumnName = serverColumnName; + } + + public String getVanColumnName() { + return vanColumnName; + } + + public void setVanColumnName(String vanColumnName) { + this.vanColumnName = vanColumnName; + } + + public String getVanAutoIncColumnName() { + return vanAutoIncColumnName; + } + + public void setVanAutoIncColumnName(String vanAutoIncColumnName) { + this.vanAutoIncColumnName = vanAutoIncColumnName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + + public Integer getSyncOrder() { + return syncOrder; + } + + public void setSyncOrder(Integer syncOrder) { + this.syncOrder = syncOrder; + } + + public Boolean getIsActive() { + return isActive; + } + + public void setIsActive(Boolean isActive) { + this.isActive = isActive; + } + + public Integer getVanID() { + return vanID; + } + + public void setVanID(Integer vanID) { + this.vanID = vanID; + } + + public Integer getProviderServiceMapID() { + return providerServiceMapID; + } + + public void setProviderServiceMapID(Integer providerServiceMapID) { + this.providerServiceMapID = providerServiceMapID; + } +} diff --git a/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java new file mode 100644 index 00000000..683bb10a --- /dev/null +++ b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java @@ -0,0 +1,41 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.repo.syncActivity_syncLayer; + +import java.util.ArrayList; + +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; + +@Repository +public interface DownSyncTableDetailRepo extends CrudRepository { + + @Query(" SELECT d FROM DownSyncTableDetail d WHERE d.isActive = true ORDER BY d.syncOrder, d.downSyncTableDetailID ") + ArrayList getActiveDownSyncTables(); + + @Query(" SELECT d FROM DownSyncTableDetail d WHERE d.isActive = true AND lower(d.tableName) = lower(:tableName) ") + ArrayList getActiveDownSyncTableByName(@Param("tableName") String tableName); +} From 9815769b2454b126860408aac0dde78efb8e7154 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Mon, 24 Aug 2026 16:47:08 +0530 Subject: [PATCH 14/81] fix: downsync implementation --- .../dataSyncActivity/StartSyncActivity.java | 49 ++ .../MMUDataSyncVanToServer.java | 52 ++ .../DownSyncDataDigester.java | 146 +++++ .../DownSyncRecordAck.java | 94 ++++ .../DownSyncTableDetail.java | 12 + .../dataSyncActivity/DataSyncRepository.java | 65 +++ .../DownSyncDataFromServer.java | 32 ++ .../DownSyncDataFromServerImpl.java | 522 ++++++++++++++++++ .../DataSyncRepositoryCentralDownload.java | 116 +++- .../GetDataFromVanAndSyncToDBImpl.java | 35 ++ .../GetDownSyncDataFromCentral.java | 35 ++ .../GetDownSyncDataFromCentralImpl.java | 125 +++++ 12 files changed, 1282 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 83c5dcfe..8b82426c 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -34,6 +34,7 @@ import org.springframework.web.bind.annotation.RestController; import com.google.gson.Gson; +import com.iemr.mmu.service.dataSyncActivity.DownSyncDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl; import com.iemr.mmu.service.dataSyncActivity.UploadDataToServerImpl; @@ -58,6 +59,8 @@ public class StartSyncActivity { private DownloadDataFromServerImpl downloadDataFromServerImpl; @Autowired private DownloadDataFromServerTransactionalImpl downloadDataFromServerTransactionalImpl; + @Autowired + private DownSyncDataFromServerImpl downSyncDataFromServerImpl; private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; @@ -225,4 +228,50 @@ public String downloadTransactionToLocal(@RequestBody String requestOBJ, return response.toString(); } + + @Operation(summary = "Start the down-sync of masters & transactional data from central to local") + @PostMapping(value = { "/startDownSync" }) + public String startDownSync(@RequestBody String requestOBJ, + @RequestHeader(value = "Authorization") String authorization, + @RequestHeader(value = "ServerAuthorization") String serverAuthorization, HttpServletRequest request) { + OutputResponse response = new OutputResponse(); + try { + String jwtToken = CookieUtil.getJwtTokenFromCookie(request); + + JSONObject obj = new JSONObject(requestOBJ); + if (obj.has("vanID") && obj.get("vanID") != null) { + Integer providerServiceMapID = obj.has(PROVIDER_SERVICE_MAP_ID) && !obj.isNull(PROVIDER_SERVICE_MAP_ID) + ? obj.getInt(PROVIDER_SERVICE_MAP_ID) + : null; + + String s = downSyncDataFromServerImpl.startDownSync(serverAuthorization, jwtToken, obj.getInt("vanID"), + providerServiceMapID); + + if (s != null && s.equalsIgnoreCase("inProgress")) + response.setError(5000, "Down-sync is already in progress, kindly wait for it to finish"); + else + response.setResponse(s); + } else { + response.setError(5000, "vanID is missing, Kindly contact the administrator."); + } + } catch (Exception e) { + logger.error("Error in down-sync : " + e); + response.setError(e); + } + return response.toString(); + } + + @Operation(summary = "Down-sync progress check") + @GetMapping(value = { "/checkDownSyncProgress" }) + public String checkDownSyncProgress() { + OutputResponse response = new OutputResponse(); + try { + response.setResponse(new Gson().toJson(downSyncDataFromServerImpl.getDownSyncStatus())); + } catch (Exception e) { + logger.error("Error in down-sync progress check : " + e); + response.setError(e); + } + return response.toString(); + } + } diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java index ea943d0e..35fb0022 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java @@ -32,10 +32,12 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; import com.iemr.mmu.data.syncActivity_syncLayer.SyncDownloadMaster; import com.iemr.mmu.data.syncActivity_syncLayer.SyncUploadDataDigester; import com.iemr.mmu.service.dataSyncLayerCentral.FetchDownloadDataImpl; import com.iemr.mmu.service.dataSyncLayerCentral.GetDataFromVanAndSyncToDBImpl; +import com.iemr.mmu.service.dataSyncLayerCentral.GetDownSyncDataFromCentralImpl; import com.iemr.mmu.service.dataSyncLayerCentral.GetMasterDataFromCentralForVanImpl; import com.iemr.mmu.utils.CookieUtil; import com.iemr.mmu.utils.response.OutputResponse; @@ -57,6 +59,8 @@ public class MMUDataSyncVanToServer { private GetMasterDataFromCentralForVanImpl getMasterDataFromCentralForVanImpl; @Autowired private FetchDownloadDataImpl fetchDownloadDataImpl; + @Autowired + private GetDownSyncDataFromCentralImpl getDownSyncDataFromCentralImpl; @Operation(summary = "Sync data from van-to-server") @PostMapping(value = { "/van-to-server" }, consumes = "application/json", produces = "application/json") @@ -156,4 +160,52 @@ public String updateProcessedFlagPostDownload(@RequestBody SyncUploadDataDigeste return response.toStringWithSerialization(); } + + @Operation(summary = "Down-sync data of one configured table from central to van") + @PostMapping(value = { "/server-to-van-downsync" }, consumes = "application/json", produces = "application/json") + public String downSyncDataFromServer(@RequestBody DownSyncDataDigester downSyncDataDigester, + @RequestHeader(value = "Authorization") String Authorization) { + OutputResponse response = new OutputResponse(); + try { + if (downSyncDataDigester != null) { + String s = getDownSyncDataFromCentralImpl.getDownSyncDataForVan(downSyncDataDigester); + if (s != null) + response.setResponse(s); + else + response.setError(5000, "Error in down-sync for table " + downSyncDataDigester.getSchemaName() + "." + + downSyncDataDigester.getTableName()); + } else { + response.setError(5000, "Invalid request"); + } + } catch (Exception e) { + if (downSyncDataDigester != null) + logger.error(e + " - Error in down-sync for table " + downSyncDataDigester.getSchemaName() + "." + + downSyncDataDigester.getTableName()); + response.setError(e); + } + return response.toStringWithSerialization(); + } + + @Operation(summary = "Update the down-sync flag at central post delivery to the van") + @PostMapping(value = { + "/updateDownSyncFlagPostDownload" }, consumes = "application/json", produces = "application/json") + public String updateDownSyncFlagPostDownload(@RequestBody DownSyncDataDigester downSyncDataDigester, + @RequestHeader(value = "Authorization") String Authorization) { + OutputResponse response = new OutputResponse(); + try { + if (downSyncDataDigester != null) { + int i = getDownSyncDataFromCentralImpl.updateDownSyncFlagPostDownload(downSyncDataDigester); + response.setResponse(String.valueOf(i)); + } else { + response.setError(5000, "Invalid request"); + } + } catch (Exception e) { + if (downSyncDataDigester != null) + logger.error(e + " - Error while updating the down-sync flag for " + + downSyncDataDigester.getSchemaName() + "." + downSyncDataDigester.getTableName()); + response.setError(e); + } + return response.toStringWithSerialization(); + } + } diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java new file mode 100644 index 00000000..0a9d8fc0 --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java @@ -0,0 +1,146 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import java.util.List; + +import com.google.gson.annotations.Expose; + +public class DownSyncDataDigester { + + @Expose + private String schemaName; + @Expose + private String tableName; + /** columns to SELECT from central */ + @Expose + private String serverColumnName; + /** primary key column of the table (same name in central & local) */ + @Expose + private String vanAutoIncColumnName; + /** MASTER / TRANSACTIONAL */ + @Expose + private String tableType; + @Expose + private Integer vanID; + @Expose + private Integer providerServiceMapID; + /** populated only on the acknowledgement call */ + @Expose + private List records; + + public DownSyncDataDigester() { + } + + public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, Integer vanID, + Integer providerServiceMapID) { + DownSyncDataDigester digester = new DownSyncDataDigester(); + digester.schemaName = tableDetail.getSchemaName(); + digester.tableName = tableDetail.getTableName(); + digester.serverColumnName = tableDetail.getServerColumnName(); + digester.vanAutoIncColumnName = tableDetail.getVanAutoIncColumnName(); + digester.tableType = tableDetail.getTableType(); + digester.vanID = vanID; + digester.providerServiceMapID = providerServiceMapID; + return digester; + } + + public static DownSyncDataDigester forAck(DownSyncTableDetail tableDetail, Integer vanID, + List records) { + DownSyncDataDigester digester = new DownSyncDataDigester(); + digester.schemaName = tableDetail.getSchemaName(); + digester.tableName = tableDetail.getTableName(); + digester.vanAutoIncColumnName = tableDetail.getVanAutoIncColumnName(); + digester.tableType = tableDetail.getTableType(); + digester.vanID = vanID; + digester.records = records; + return digester; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getServerColumnName() { + return serverColumnName; + } + + public void setServerColumnName(String serverColumnName) { + this.serverColumnName = serverColumnName; + } + + public String getVanAutoIncColumnName() { + return vanAutoIncColumnName; + } + + public void setVanAutoIncColumnName(String vanAutoIncColumnName) { + this.vanAutoIncColumnName = vanAutoIncColumnName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + + public Integer getVanID() { + return vanID; + } + + public void setVanID(Integer vanID) { + this.vanID = vanID; + } + + public Integer getProviderServiceMapID() { + return providerServiceMapID; + } + + public void setProviderServiceMapID(Integer providerServiceMapID) { + this.providerServiceMapID = providerServiceMapID; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + public boolean isMasterTable() { + return tableType == null || DownSyncTableDetail.TABLE_TYPE_MASTER.equalsIgnoreCase(tableType); + } +} diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java new file mode 100644 index 00000000..5be35f81 --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java @@ -0,0 +1,94 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import com.google.gson.annotations.Expose; + +public class DownSyncRecordAck { + + public static final String STATUS_PROCESSED = "P"; + public static final String STATUS_FAILED = "F"; + public static final String CONFLICT = "CONFLICT"; + + @Expose + private Long centralID; + @Expose + private Long vanSerialNo; + @Expose + private String status; + @Expose + private String failureReason; + + public DownSyncRecordAck() { + } + + public DownSyncRecordAck(Long centralID, Long vanSerialNo, String status, String failureReason) { + this.centralID = centralID; + this.vanSerialNo = vanSerialNo; + this.status = status; + this.failureReason = failureReason; + } + + public static DownSyncRecordAck success(Long centralID, Long vanSerialNo) { + return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_PROCESSED, null); + } + + public static DownSyncRecordAck failure(Long centralID, Long vanSerialNo, String failureReason) { + return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_FAILED, failureReason); + } + + public boolean isSuccess() { + return STATUS_PROCESSED.equalsIgnoreCase(status); + } + + public Long getCentralID() { + return centralID; + } + + public void setCentralID(Long centralID) { + this.centralID = centralID; + } + + public Long getVanSerialNo() { + return vanSerialNo; + } + + public void setVanSerialNo(Long vanSerialNo) { + this.vanSerialNo = vanSerialNo; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getFailureReason() { + return failureReason; + } + + public void setFailureReason(String failureReason) { + this.failureReason = failureReason; + } +} diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index c2f18050..85e70fe6 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -69,6 +69,9 @@ public class DownSyncTableDetail { @Expose @Column(name = "TableType") private String tableType; + + @Column(name = "LastModColumnName") + private String lastModColumnName; /** lower number syncs first - enforces the FK dependency chain */ @Expose @Column(name = "SyncOrder") @@ -135,6 +138,15 @@ public void setVanColumnName(String vanColumnName) { this.vanColumnName = vanColumnName; } + public String getLastModColumnName() { + return (lastModColumnName == null || lastModColumnName.trim().isEmpty()) ? "LastModDate" + : lastModColumnName.trim(); + } + + public void setLastModColumnName(String lastModColumnName) { + this.lastModColumnName = lastModColumnName; + } + public String getVanAutoIncColumnName() { return vanAutoIncColumnName; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index dc36c7c1..6bbc33f8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.Map; +import java.sql.PreparedStatement; + import javax.sql.DataSource; import org.slf4j.Logger; @@ -33,6 +35,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.CreatedDate; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Service; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo; @@ -137,4 +141,65 @@ public int[] updateLatestMasterInLocal(String query, List syncDataList // ---------------------------------- End of Download Repository + // ---------------------------------- Down-Sync Repository (central -> local) + + public List getDownSyncColumns(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + + " AND COLUMN_NAME NOT IN ('DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " + + " 'LastDownSyncDate') " + " ORDER BY COLUMN_NAME "; + + return jdbcTemplate.queryForList(query, String.class, schema, table); + } + + public Map getLocalRecordForDownSync(String schema, String table, String autoIncColumnName, + Object vanSerialNo, Object vanID, String lastModColumn) { + if (vanSerialNo == null) + return null; + + jdbcTemplate = getJdbcTemplate(); + String query = " SELECT " + autoIncColumnName + ", Processed, " + lastModColumn + + " AS LastModDate, LastDownSyncDate FROM " + schema + "." + table + + " WHERE " + autoIncColumnName + " = ? AND VanID = ? "; + + List> resultSet = jdbcTemplate.queryForList(query, vanSerialNo, vanID); + if (resultSet == null || resultSet.isEmpty()) + return null; + + return resultSet.get(0); + } + + public Long insertDownSyncRecordInLocal(final String query, final Object[] params) { + jdbcTemplate = getJdbcTemplate(); + KeyHolder keyHolder = new GeneratedKeyHolder(); + + jdbcTemplate.update(connection -> { + PreparedStatement ps = connection.prepareStatement(query, PreparedStatement.RETURN_GENERATED_KEYS); + for (int i = 0; i < params.length; i++) { + ps.setObject(i + 1, params[i]); + } + return ps; + }, keyHolder); + + Number key = keyHolder.getKey(); + return key != null ? key.longValue() : null; + } + + public int updateDownSyncRecordInLocal(String query, Object[] params) { + jdbcTemplate = getJdbcTemplate(); + return jdbcTemplate.update(query, params); + } + + public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { + jdbcTemplate = getJdbcTemplate(); + String query = " UPDATE " + schema + "." + table + + " SET Processed = 'F', SyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; + return jdbcTemplate.update(query, localID); + } + + // End of Down-Sync Repository + + } \ No newline at end of file diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java new file mode 100644 index 00000000..3fcabcf8 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java @@ -0,0 +1,32 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import java.util.Map; + +public interface DownSyncDataFromServer { + + String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, Integer providerServiceMapID) + throws Exception; + + Map getDownSyncStatus(); +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java new file mode 100644 index 00000000..632af829 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -0,0 +1,522 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import java.sql.Timestamp; +import java.text.DateFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; +import com.iemr.mmu.repo.syncActivity_syncLayer.DownSyncTableDetailRepo; +import com.iemr.mmu.utils.RestTemplateUtil; + + +@Service +@PropertySource("classpath:application.properties") +public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { + + private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private static final String VAN_SERIAL_NO = "VanSerialNo"; + private static final String VAN_ID = "VanID"; + private static final String LAST_MOD_DATE = "LastModDate"; + private static final String LAST_DOWN_SYNC_DATE = "LastDownSyncDate"; + private static final String DOWN_SYNCED = "DownSynced"; + private static final String DOWN_SYNC_DATE = "DownSyncDate"; + private static final String DOWN_SYNC_FAILURE_REASON = "DownSyncFailureReason"; + private static final String PROCESSED = "Processed"; + private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; + + @Value("${downSyncDataUrl}") + private String downSyncDataUrl; + + @Value("${downSyncFlagUpdateUrl}") + private String downSyncFlagUpdateUrl; + + @Autowired + private DownSyncTableDetailRepo downSyncTableDetailRepo; + + @Autowired + private DataSyncRepository dataSyncRepository; + + private static final AtomicBoolean IN_PROGRESS = new AtomicBoolean(false); + private static int totalCounter = 0; + private static int progressCounter = 0; + private static int insertedCounter = 0; + private static int updatedCounter = 0; + private static int conflictCounter = 0; + private static int skippedCounter = 0; + private static int failedCounter = 0; + private static StringBuilder failedTables = new StringBuilder(); + private static String currentTable = ""; + + public String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, + Integer providerServiceMapID) throws Exception { + + if (vanID == null) + throw new Exception("vanID is mandatory for down-sync. Kindly contact the administrator."); + + final ArrayList downSyncTables = downSyncTableDetailRepo.getActiveDownSyncTables(); + if (downSyncTables == null || downSyncTables.isEmpty()) + throw new Exception("No table is configured for down-sync in m_downsynctabledetail."); + + if (!IN_PROGRESS.compareAndSet(false, true)) + return "inProgress"; + + totalCounter = downSyncTables.size(); + progressCounter = 0; + insertedCounter = 0; + updatedCounter = 0; + conflictCounter = 0; + skippedCounter = 0; + failedCounter = 0; + failedTables = new StringBuilder(); + currentTable = ""; + + final ExecutorService threadPool = Executors.newSingleThreadExecutor(); + try { + threadPool.submit(() -> { + try { + for (DownSyncTableDetail tableDetail : downSyncTables) { + currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); + try { + downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); + } catch (Exception e) { + failedCounter++; + failedTables.append(tableDetail.getTableName()).append(" | "); + logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); + } + progressCounter++; + } + } finally { + IN_PROGRESS.set(false); + } + return "Down-sync completed"; + }); + } finally { + threadPool.shutdown(); + } + + return " Down-sync started "; + } + + public Map getDownSyncStatus() { + Map resultMap = new LinkedHashMap<>(); + resultMap.put("percentage", totalCounter == 0 ? 0 : Math.floor((progressCounter * 100) / totalCounter)); + resultMap.put("inProgress", IN_PROGRESS.get()); + resultMap.put("currentTable", currentTable); + resultMap.put("totalTables", totalCounter); + resultMap.put("completedTables", progressCounter); + resultMap.put("failedTableCount", failedCounter); + resultMap.put("failedTables", failedTables.toString()); + resultMap.put("recordsInserted", insertedCounter); + resultMap.put("recordsUpdated", updatedCounter); + resultMap.put("recordsSkipped", skippedCounter); + resultMap.put("conflicts", conflictCounter); + return resultMap; + } + + /*** + * Down-syncs one configured table. + */ + private void downSyncTable(DownSyncTableDetail tableDetail, Integer vanID, Integer providerServiceMapID, + String serverAuthorization, String jwtToken) throws Exception { + + List serverColumns = splitColumns(tableDetail.getServerColumnName()); + List vanColumns = splitColumns(tableDetail.getVanColumnName()); + + if (serverColumns.isEmpty() && vanColumns.isEmpty()) { + List resolved = dataSyncRepository.getDownSyncColumns(tableDetail.getSchemaName(), + tableDetail.getTableName()); + if (resolved.isEmpty()) + throw new Exception("No column found locally for " + tableDetail.getSchemaName() + "." + + tableDetail.getTableName() + + ". The table is configured for down-sync but does not exist in the local DB."); + + serverColumns = resolved; + vanColumns = resolved; + } + + if (serverColumns.isEmpty() || vanColumns.isEmpty()) + throw new Exception("Only one of ServerColumnName / VanColumnName is configured for " + + tableDetail.getTableName() + ". Configure both, with the same columns in the same order, " + + "or leave both empty to have them resolved from the schema."); + if (serverColumns.size() != vanColumns.size()) + throw new Exception("ServerColumnName & VanColumnName column count does not match for " + + tableDetail.getTableName() + " (" + serverColumns.size() + " vs " + vanColumns.size() + + "). Central and local must expose the same number of columns, in the same order."); + + List> dataFromCentral = fetchDataFromCentral(tableDetail, String.join(",", serverColumns), + vanID, providerServiceMapID, serverAuthorization, jwtToken); + + if (dataFromCentral.isEmpty()) { + logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), tableDetail.getTableName()); + return; + } + + if (tableDetail.isMasterTable()) { + upsertMasterData(tableDetail, serverColumns, vanColumns, dataFromCentral); + } else { + List acks = saveTransactionalData(tableDetail, serverColumns, vanColumns, + dataFromCentral, vanID); + acknowledgeToCentral(tableDetail, vanID, acks, serverAuthorization, jwtToken); + } + } + + private List> fetchDataFromCentral(DownSyncTableDetail tableDetail, String serverColumnName, + Integer vanID, Integer providerServiceMapID, String serverAuthorization, String jwtToken) + throws Exception { + + DownSyncDataDigester digester = DownSyncDataDigester.forDownload(tableDetail, serverColumnName, vanID, + providerServiceMapID); + + RestTemplate restTemplate = new RestTemplate(); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + ResponseEntity response = restTemplate.exchange(downSyncDataUrl, HttpMethod.POST, request, String.class); + + if (response == null || !response.hasBody()) + throw new Exception("Empty response from central for " + tableDetail.getTableName()); + + JSONObject responseObj = new JSONObject(response.getBody()); + if (!responseObj.has("data") || !responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) + throw new Exception("Down-sync API failed for " + tableDetail.getTableName() + " : " + + responseObj.optString("errorMessage")); + + Gson gson = new GsonBuilder().serializeNulls().create(); + List> data = gson.fromJson(responseObj.get("data").toString(), + new TypeToken>>() { + }.getType()); + + return data != null ? data : new ArrayList<>(); + } + + /*** + * MASTER tables : central authoritative, upserted in one batch. + */ + private void upsertMasterData(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, + List> dataFromCentral) { + + StringBuilder placeHolders = new StringBuilder(); + StringBuilder onDuplicate = new StringBuilder(); + for (int i = 0; i < vanColumns.size(); i++) { + if (i > 0) { + placeHolders.append(", "); + onDuplicate.append(", "); + } + placeHolders.append("?"); + onDuplicate.append(vanColumns.get(i)).append(" = VALUES(").append(vanColumns.get(i)).append(")"); + } + + String query = " INSERT INTO " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " ( " + + String.join(", ", vanColumns) + " ) VALUES ( " + placeHolders + " ) ON DUPLICATE KEY UPDATE " + + onDuplicate; + + List batch = new ArrayList<>(); + for (Map record : dataFromCentral) { + Object[] values = new Object[serverColumns.size()]; + for (int i = 0; i < serverColumns.size(); i++) { + values[i] = normalize(record.get(serverColumns.get(i))); + } + batch.add(values); + } + + int[] result = dataSyncRepository.updateLatestMasterInLocal(query, batch); + updatedCounter += result != null ? result.length : 0; + logger.info("Down-synced {} master records into {}.{}", batch.size(), tableDetail.getSchemaName(), + tableDetail.getTableName()); + } + + /*** + * TRANSACTIONAL tables : record by record, so that a conflict on one record + * does not block the delivery of the rest. + */ + private List saveTransactionalData(DownSyncTableDetail tableDetail, List serverColumns, + List vanColumns, List> dataFromCentral, Integer vanID) throws Exception { + + String pkColumn = tableDetail.getVanAutoIncColumnName(); + if (pkColumn == null || pkColumn.trim().isEmpty()) + throw new Exception( + "VanAutoIncColumnName is not configured for transactional table " + tableDetail.getTableName()); + pkColumn = pkColumn.trim(); + + requireIdentifier(pkColumn, "VanAutoIncColumnName", tableDetail); + requireIdentifier(tableDetail.getLastModColumnName(), "LastModColumnName", tableDetail); + + List acks = new ArrayList<>(); + + for (Map record : dataFromCentral) { + Long centralID = toLong(record.get(pkColumn)); + Object vanSerialNo = normalize(record.get(VAN_SERIAL_NO)); + + try { + Map localRecord = dataSyncRepository.getLocalRecordForDownSync( + tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, vanSerialNo, vanID, + tableDetail.getLastModColumnName()); + + if (localRecord == null) { + Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); + insertedCounter++; + acks.add(DownSyncRecordAck.success(centralID, localID)); + continue; + } + + Long localID = toLong(localRecord.get(pkColumn)); + + if (!isCentralCopyNewer(record, localRecord, tableDetail.getLastModColumnName())) { + skippedCounter++; + acks.add(DownSyncRecordAck.success(centralID, localID)); + continue; + } + + if (hasUnsentLocalChanges(localRecord)) { + dataSyncRepository.markDownSyncConflictInLocal(tableDetail.getSchemaName(), + tableDetail.getTableName(), pkColumn, localID); + conflictCounter++; + acks.add(DownSyncRecordAck.failure(centralID, localID, DownSyncRecordAck.CONFLICT)); + logger.warn("Down-sync conflict for {}.{}, local id {}", tableDetail.getSchemaName(), + tableDetail.getTableName(), localID); + continue; + } + + updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, localID); + updatedCounter++; + acks.add(DownSyncRecordAck.success(centralID, localID)); + + } catch (Exception e) { + failedCounter++; + String reason = e.getMessage() != null && e.getMessage().length() > 250 + ? e.getMessage().substring(0, 250) + : e.getMessage(); + acks.add(DownSyncRecordAck.failure(centralID, toLong(vanSerialNo), reason)); + logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + + e.getMessage(), e); + } + } + + return acks; + } + + private boolean isCentralCopyNewer(Map centralRecord, Map localRecord, + String lastModColumn) { + Timestamp centralLastModDate = toTimestamp(centralRecord.get(lastModColumn)); + if (centralLastModDate == null) + return false; + + Timestamp localLastModDate = toTimestamp(localRecord.get(LAST_MOD_DATE)); + if (localLastModDate != null) + return centralLastModDate.after(localLastModDate); + + Timestamp localLastDownSyncDate = toTimestamp(localRecord.get(LAST_DOWN_SYNC_DATE)); + return localLastDownSyncDate == null || centralLastModDate.after(localLastDownSyncDate); + } + + private boolean hasUnsentLocalChanges(Map localRecord) { + String localProcessed = localRecord.get(PROCESSED) != null ? String.valueOf(localRecord.get(PROCESSED)) : null; + return "N".equalsIgnoreCase(localProcessed); + } + + private Long insertRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, + String pkColumn, Map record) { + + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + + for (int i = 0; i < vanColumns.size(); i++) { + String vanColumn = vanColumns.get(i); + // the local auto-increment PK is skipped so that local generates its own value + if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + continue; + + columns.add(vanColumn); + values.add(normalize(record.get(serverColumns.get(i)))); + } + + columns.add(PROCESSED); + values.add("P"); + columns.add(SYNC_FAILURE_REASON); + values.add(null); + columns.add(LAST_DOWN_SYNC_DATE); + values.add(Timestamp.valueOf(LocalDateTime.now())); + + StringBuilder placeHolders = new StringBuilder(); + for (int i = 0; i < columns.size(); i++) { + if (i > 0) + placeHolders.append(", "); + placeHolders.append("?"); + } + + String query = " INSERT INTO " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " ( " + + String.join(", ", columns) + " ) VALUES ( " + placeHolders + " ) "; + + return dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + } + + private void updateRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, + String pkColumn, Map record, Long localID) { + + StringBuilder setClause = new StringBuilder(); + List values = new ArrayList<>(); + + for (int i = 0; i < vanColumns.size(); i++) { + String vanColumn = vanColumns.get(i); + if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + continue; + + if (setClause.length() > 0) + setClause.append(", "); + setClause.append(vanColumn).append(" = ?"); + values.add(normalize(record.get(serverColumns.get(i)))); + } + + setClause.append(setClause.length() > 0 ? ", " : "").append(PROCESSED).append(" = 'P', ") + .append(SYNC_FAILURE_REASON).append(" = NULL, ").append(LAST_DOWN_SYNC_DATE).append(" = ? "); + values.add(Timestamp.valueOf(LocalDateTime.now())); + values.add(localID); + + String query = " UPDATE " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " SET " + setClause + + " WHERE " + pkColumn + " = ? "; + + dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); + } + + private void requireIdentifier(String value, String what, DownSyncTableDetail tableDetail) throws Exception { + if (value == null || !value.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + throw new Exception("The down-sync configuration of " + tableDetail.getTableName() + " holds an invalid " + + what); + } + + private boolean isDownSyncManagedColumn(String column) { + return PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) + || LAST_DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNCED.equalsIgnoreCase(column) + || DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNC_FAILURE_REASON.equalsIgnoreCase(column); + } + + private void acknowledgeToCentral(DownSyncTableDetail tableDetail, Integer vanID, List acks, + String serverAuthorization, String jwtToken) throws Exception { + + if (acks == null || acks.isEmpty()) + return; + + DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, acks); + + RestTemplate restTemplate = new RestTemplate(); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + ResponseEntity response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, + String.class); + + if (response == null || !response.hasBody()) + throw new Exception("Empty response while updating the down-sync flag for " + tableDetail.getTableName()); + + JSONObject responseObj = new JSONObject(response.getBody()); + if (!responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) + throw new Exception("Could not update the down-sync flag at central for " + tableDetail.getTableName() + + " : " + responseObj.optString("errorMessage")); + + logger.info("Down-sync flag updated at central for {} records of {}", acks.size(), tableDetail.getTableName()); + } + + private List splitColumns(String columns) { + List columnList = new ArrayList<>(); + if (columns == null) + return columnList; + + for (String column : Arrays.asList(columns.split(","))) { + if (column != null && !column.trim().isEmpty()) + columnList.add(column.trim()); + } + return columnList; + } + + private Object normalize(Object value) { + if (value instanceof Double) { + double doubleValue = (Double) value; + if (!Double.isInfinite(doubleValue) && !Double.isNaN(doubleValue) && doubleValue == Math.floor(doubleValue) + && Math.abs(doubleValue) < 1e15) + return (long) doubleValue; + } + return value; + } + + private Long toLong(Object value) { + Object normalized = normalize(value); + if (normalized == null) + return null; + if (normalized instanceof Number) + return ((Number) normalized).longValue(); + try { + return Long.valueOf(String.valueOf(normalized).trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private Timestamp toTimestamp(Object value) { + if (value == null) + return null; + if (value instanceof Timestamp) + return (Timestamp) value; + if (value instanceof java.util.Date) + return new Timestamp(((java.util.Date) value).getTime()); + + String stringValue = String.valueOf(value).trim(); + if (stringValue.isEmpty() || "null".equalsIgnoreCase(stringValue)) + return null; + + String candidate = stringValue.replace("T", " ").replace("Z", ""); + try { + return Timestamp.valueOf(candidate); + } catch (IllegalArgumentException e) { + try { + // gson's default rendering of a java.sql.Timestamp + return new Timestamp(DateFormat.getDateTimeInstance().parse(stringValue).getTime()); + } catch (java.text.ParseException pe) { + logger.warn("Could not parse the timestamp '{}' received from central", stringValue); + return null; + } + } + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 565e2466..2cceb6d8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -28,6 +28,8 @@ import javax.sql.DataSource; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -54,6 +56,8 @@ private JdbcTemplate getJdbcTemplate() { private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private static final String DOWN_SYNC_TRANSACTIONAL = "TRANSACTIONAL"; + // Data Upload Repository public int checkRecordIsAlreadyPresentOrNot(String schemaName, String tableName, String vanSerialNo, String vanID, String vanAutoIncColumnName, int syncFacilityID) { @@ -205,4 +209,114 @@ else if (masterType.equalsIgnoreCase("P")) { } // End of Data Download Repository -} \ No newline at end of file + + // ---------------------------------- Down-Sync Repository (central -> local) + + public List> getDownSyncDataFromTable(String schema, String table, String columnNames, + String tableType, Integer vanID, String lastModColumn) throws Exception { + jdbcTemplate = getJdbcTemplate(); + + if (schema == null || table == null) + throw new Exception("Invalid down-sync request. Schema/table info is missing"); + + String columns = (columnNames == null || columnNames.trim().isEmpty()) ? " * " : columnNames; + List> resultSetList; + + if (DOWN_SYNC_TRANSACTIONAL.equalsIgnoreCase(tableType)) { + if (vanID == null) + throw new Exception("Invalid down-sync request. VanID is mandatory for transactional table " + table); + + String query = " SELECT " + columns + " FROM " + schema + "." + table + + " WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') " + + " OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND " + lastModColumn + + " > DownSyncDate ) ) "; + logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + resultSetList = jdbcTemplate.queryForList(query, vanID); + } else { + String query = " SELECT " + columns + " FROM " + schema + "." + table; + logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + resultSetList = jdbcTemplate.queryForList(query); + } + + logger.info("Down-sync record count for {}.{} : {}", schema, table, resultSetList.size()); + return resultSetList; + } + + public int updateDownSyncFlagPostDownload(String schema, String table, String pkColumnName, + List records) throws Exception { + jdbcTemplate = getJdbcTemplate(); + + if (schema == null || table == null || pkColumnName == null) + throw new Exception("Invalid request. Schema/table/primary key info is missing"); + if (records == null || records.isEmpty()) + return 0; + + List successWithSerialNo = new ArrayList<>(); + List successWithoutSerialNo = new ArrayList<>(); + List failed = new ArrayList<>(); + + for (DownSyncRecordAck record : records) { + if (record == null || record.getCentralID() == null) + continue; + + if (record.isSuccess()) { + if (record.getVanSerialNo() != null) + successWithSerialNo.add(new Object[] { record.getVanSerialNo(), record.getCentralID() }); + else + successWithoutSerialNo.add(new Object[] { record.getCentralID() }); + } else { + failed.add(new Object[] { record.getFailureReason(), record.getCentralID() }); + } + } + + int updatedRows = 0; + + if (!successWithSerialNo.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL, VanSerialNo = ? " + + " WHERE " + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithSerialNo)); + } + + if (!successWithoutSerialNo.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE " + + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); + } + + if (!failed.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, failed)); + } + + logger.info("Down-sync flag updated for {}.{}. Records : {}, rows : {}", schema, table, records.size(), + updatedRows); + return updatedRows; + } + + public int markDownSyncedPostUpSync(String schema, String table, List vanSerialNoAndVanID) { + if (schema == null || table == null || vanSerialNoAndVanID == null || vanSerialNoAndVanID.isEmpty()) + return 0; + + jdbcTemplate = getJdbcTemplate(); + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + + " WHERE VanSerialNo = ? AND VanID = ? "; + return countUpdatedRows(jdbcTemplate.batchUpdate(query, vanSerialNoAndVanID)); + } + + private int countUpdatedRows(int[] batchResult) { + int count = 0; + if (batchResult != null) { + for (int rows : batchResult) { + if (rows > 0) + count += rows; + } + } + return count; + } + + // End of Down-Sync Repository +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java index cca5634d..3205007c 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; @@ -46,6 +47,12 @@ public class GetDataFromVanAndSyncToDBImpl implements GetDataFromVanAndSyncToDB @Autowired private DataSyncRepositoryCentral dataSyncRepositoryCentral; + @Autowired + private GetDownSyncDataFromCentral getDownSyncDataFromCentral; + + @Value("${downSync.markDeliveredOnUpSync:true}") + private boolean markDownSyncedOnUpSync; + private static final Map> TABLE_GROUPS = new HashMap<>(); static { TABLE_GROUPS.put(1, @@ -326,6 +333,7 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig Map insertIndexMap = new HashMap<>(); Map updateIndexMap = new HashMap<>(); + Map vanIDByResultIndex = new HashMap<>(); boolean overallSuccess = true; @@ -463,6 +471,7 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig int currentSyncResultIndex = syncResults.size(); syncResults.add(new SyncResult(schemaName, syncTableName, vanSerialNo, syncUploadDataDigester.getSyncedBy(), false, "Pending")); + vanIDByResultIndex.put(currentSyncResultIndex, vanID); if (recordCheck == 0) { insertIndexMap.put(currentSyncResultIndex, syncDataListInsert.size()); @@ -591,9 +600,35 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig } logger.info("Sync results for table {}: {}", syncTableName, syncResults); + + flagUpSyncedRecordsForDownSync(schemaName, syncTableName, syncResults, vanIDByResultIndex); + return overallSuccess; } + private void flagUpSyncedRecordsForDownSync(String schemaName, String tableName, List syncResults, + Map vanIDByResultIndex) { + if (!markDownSyncedOnUpSync || vanIDByResultIndex.isEmpty()) + return; + + List vanSerialNoAndVanID = new ArrayList<>(); + for (Map.Entry entry : vanIDByResultIndex.entrySet()) { + int index = entry.getKey(); + if (index >= syncResults.size()) + continue; + + SyncResult result = syncResults.get(index); + if (result == null || !result.isSuccess() || result.getVanSerialNo() == null + || entry.getValue() == null || "null".equalsIgnoreCase(entry.getValue())) + continue; + + vanSerialNoAndVanID.add(new Object[] { result.getVanSerialNo(), entry.getValue() }); + } + + if (!vanSerialNoAndVanID.isEmpty()) + getDownSyncDataFromCentral.markDownSyncedPostUpSync(schemaName, tableName, vanSerialNoAndVanID); + } + private String getVanSerialNo(Object[] record, int vanSerialIndex, SyncResult originalResult) { if (vanSerialIndex >= 0 && vanSerialIndex < record.length) { return String.valueOf(record[vanSerialIndex]); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java new file mode 100644 index 00000000..553ee73e --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java @@ -0,0 +1,35 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.util.List; + +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; + +public interface GetDownSyncDataFromCentral { + + String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) throws Exception; + + int updateDownSyncFlagPostDownload(DownSyncDataDigester downSyncDataDigester) throws Exception; + + int markDownSyncedPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID); +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java new file mode 100644 index 00000000..42e71dc6 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -0,0 +1,125 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; +import com.iemr.mmu.repo.syncActivity_syncLayer.DownSyncTableDetailRepo; + + +@Service +public class GetDownSyncDataFromCentralImpl implements GetDownSyncDataFromCentral { + + private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + @Autowired + private DataSyncRepositoryCentralDownload dataSyncRepositoryCentralDownload; + + @Autowired + private DownSyncTableDetailRepo downSyncTableDetailRepo; + + public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) throws Exception { + if (downSyncDataDigester == null || downSyncDataDigester.getSchemaName() == null + || downSyncDataDigester.getTableName() == null) + throw new Exception("Invalid down-sync request. Either schema or table info is missing/wrong"); + + if (!downSyncDataDigester.isMasterTable() && downSyncDataDigester.getVanID() == null) + throw new Exception("Invalid down-sync request. VanID is mandatory for a transactional table"); + + List> resultSetList = dataSyncRepositoryCentralDownload.getDownSyncDataFromTable( + downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), + downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), + downSyncDataDigester.getVanID()); + + return downSyncGson().toJson(resultSetList); + } + + + private Gson downSyncGson() { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.serializeNulls(); + gsonBuilder.registerTypeAdapter(Timestamp.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive(src.toString())); + gsonBuilder.registerTypeAdapter(java.sql.Date.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive(src.toString())); + gsonBuilder.registerTypeAdapter(java.sql.Time.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive(src.toString())); + gsonBuilder.registerTypeAdapter(LocalDateTime.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive( + Timestamp.valueOf(src).toString())); + return gsonBuilder.create(); + } + + public int updateDownSyncFlagPostDownload(DownSyncDataDigester downSyncDataDigester) throws Exception { + if (downSyncDataDigester == null || downSyncDataDigester.getSchemaName() == null + || downSyncDataDigester.getTableName() == null || downSyncDataDigester.getRecords() == null) + throw new Exception("Invalid request. Schema, table or record info is missing/wrong"); + + String pkColumnName = downSyncDataDigester.getVanAutoIncColumnName(); + if (pkColumnName == null || pkColumnName.trim().isEmpty()) + throw new Exception("Invalid request. Primary key column of " + downSyncDataDigester.getTableName() + + " is missing in the request"); + + return dataSyncRepositoryCentralDownload.updateDownSyncFlagPostDownload(downSyncDataDigester.getSchemaName(), + downSyncDataDigester.getTableName(), pkColumnName.trim(), downSyncDataDigester.getRecords()); + } + + + public int resetDownSyncFlagPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID) { + if (schemaName == null || tableName == null || vanSerialNoAndVanID == null || vanSerialNoAndVanID.isEmpty()) + return 0; + + try { + ArrayList tableDetails = downSyncTableDetailRepo + .getActiveDownSyncTableByName(tableName); + boolean downSyncEnabled = false; + for (DownSyncTableDetail tableDetail : tableDetails) { + if (tableDetail.isTransactionalTable()) + downSyncEnabled = true; + } + if (!downSyncEnabled) + return 0; + + return dataSyncRepositoryCentralDownload.resetDownSyncFlagPostUpSync(schemaName, tableName, + vanSerialNoAndVanID); + } catch (Exception e) { + logger.warn("Could not reset the DownSynced flag for {}.{} post up-sync : {}", schemaName, tableName, + e.getMessage()); + return 0; + } + } +} From 1090c4b3541946e56bb54529aac3c032850ade77 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Mon, 24 Aug 2026 17:01:18 +0530 Subject: [PATCH 15/81] fix: build issue --- .../DownSyncDataDigester.java | 6 ++-- .../GetDownSyncDataFromCentralImpl.java | 28 +++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java index 0a9d8fc0..af143161 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java @@ -51,12 +51,12 @@ public class DownSyncDataDigester { public DownSyncDataDigester() { } - public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, Integer vanID, - Integer providerServiceMapID) { + public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, String serverColumnName, + Integer vanID, Integer providerServiceMapID) { DownSyncDataDigester digester = new DownSyncDataDigester(); digester.schemaName = tableDetail.getSchemaName(); digester.tableName = tableDetail.getTableName(); - digester.serverColumnName = tableDetail.getServerColumnName(); + digester.serverColumnName = serverColumnName; digester.vanAutoIncColumnName = tableDetail.getVanAutoIncColumnName(); digester.tableType = tableDetail.getTableType(); digester.vanID = vanID; diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java index 42e71dc6..54b931ac 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -63,12 +63,28 @@ public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) t List> resultSetList = dataSyncRepositoryCentralDownload.getDownSyncDataFromTable( downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), - downSyncDataDigester.getVanID()); + downSyncDataDigester.getVanID(), resolveLastModColumn(downSyncDataDigester.getTableName())); return downSyncGson().toJson(resultSetList); } - + private String resolveLastModColumn(String tableName) throws Exception { + String lastModColumn = "LastModDate"; + + if (tableName != null) { + ArrayList tableDetails = downSyncTableDetailRepo + .getActiveDownSyncTableByName(tableName.trim()); + if (tableDetails != null && !tableDetails.isEmpty()) + lastModColumn = tableDetails.get(0).getLastModColumnName(); + } + + if (lastModColumn == null || !lastModColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + throw new Exception( + "The down-sync configuration of " + tableName + " holds an invalid modification-time column name"); + + return lastModColumn; + } + private Gson downSyncGson() { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.serializeNulls(); @@ -98,8 +114,8 @@ public int updateDownSyncFlagPostDownload(DownSyncDataDigester downSyncDataDiges downSyncDataDigester.getTableName(), pkColumnName.trim(), downSyncDataDigester.getRecords()); } - - public int resetDownSyncFlagPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID) { + @Override + public int markDownSyncedPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID) { if (schemaName == null || tableName == null || vanSerialNoAndVanID == null || vanSerialNoAndVanID.isEmpty()) return 0; @@ -114,10 +130,10 @@ public int resetDownSyncFlagPostUpSync(String schemaName, String tableName, List if (!downSyncEnabled) return 0; - return dataSyncRepositoryCentralDownload.resetDownSyncFlagPostUpSync(schemaName, tableName, + return dataSyncRepositoryCentralDownload.markDownSyncedPostUpSync(schemaName, tableName, vanSerialNoAndVanID); } catch (Exception e) { - logger.warn("Could not reset the DownSynced flag for {}.{} post up-sync : {}", schemaName, tableName, + logger.warn("Could not mark the DownSynced flag for {}.{} post up-sync : {}", schemaName, tableName, e.getMessage()); return 0; } From eaf2edd060b6cfc26ba422960d850ea5fe94408c Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 25 Aug 2026 20:08:23 +0530 Subject: [PATCH 16/81] fix: lastmoddate updation --- .../syncActivity_syncLayer/DownSyncTableDetail.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index 85e70fe6..c7a0dd68 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -41,6 +41,7 @@ public class DownSyncTableDetail { public static final String TABLE_TYPE_MASTER = "MASTER"; public static final String TABLE_TYPE_TRANSACTIONAL = "TRANSACTIONAL"; + public static final String LAST_MOD_COLUMN = "LastModDate"; @Id @GeneratedValue(strategy = GenerationType.AUTO) @@ -70,8 +71,6 @@ public class DownSyncTableDetail { @Column(name = "TableType") private String tableType; - @Column(name = "LastModColumnName") - private String lastModColumnName; /** lower number syncs first - enforces the FK dependency chain */ @Expose @Column(name = "SyncOrder") @@ -139,12 +138,7 @@ public void setVanColumnName(String vanColumnName) { } public String getLastModColumnName() { - return (lastModColumnName == null || lastModColumnName.trim().isEmpty()) ? "LastModDate" - : lastModColumnName.trim(); - } - - public void setLastModColumnName(String lastModColumnName) { - this.lastModColumnName = lastModColumnName; + return LAST_MOD_COLUMN; } public String getVanAutoIncColumnName() { From 40686bd7695e2ad70d75ccedc5c8aaded40e7044 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 15:29:46 +0530 Subject: [PATCH 17/81] fix: update the response of downsync --- .../dataSyncActivity/StartSyncActivity.java | 8 + .../DownSyncTableResult.java | 159 ++++++++++++++++++ .../dataSyncActivity/DataSyncRepository.java | 23 ++- .../DownSyncDataFromServerImpl.java | 71 ++++++-- 4 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 8b82426c..baf098da 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -238,6 +238,14 @@ public String startDownSync(@RequestBody String requestOBJ, try { String jwtToken = CookieUtil.getJwtTokenFromCookie(request); + if (serverAuthorization == null || serverAuthorization.trim().isEmpty()) { + logger.error("Down-sync : ServerAuthorization is empty - do the data sync server login first"); + response.setError(5000, + "Server authorization is missing. Kindly do the data sync server login and try again."); + return response.toString(); + } + logger.info("Down-sync : ServerAuthorization received, length {}", serverAuthorization.trim().length()); + JSONObject obj = new JSONObject(requestOBJ); if (obj.has("vanID") && obj.get("vanID") != null) { Integer providerServiceMapID = obj.has(PROVIDER_SERVICE_MAP_ID) && !obj.isNull(PROVIDER_SERVICE_MAP_ID) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java new file mode 100644 index 00000000..7c54b79d --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java @@ -0,0 +1,159 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.google.gson.annotations.Expose; + +/*** + * The outcome of down-syncing one table, reported to the screen. + * + *

+ * A table can fail in two quite different ways and the difference matters when + * reading the result: + * + *

    + *
  • the table itself failed - central could not even return the rows (a schema + * mismatch, say), so nothing was delivered and no record was flagged;
    • + *
  • the table was delivered but individual records failed, in which case the + * table shows a record failure count and each distinct reason with a tally.
    • + *
+ */ +public class DownSyncTableResult { + + public static final String STATUS_SUCCESS = "SUCCESS"; + public static final String STATUS_FAILED = "FAILED"; + public static final String STATUS_PARTIAL = "PARTIAL"; + + @Expose + private String groupName; + @Expose + private String schemaName; + @Expose + private String tableName; + @Expose + private String status; + @Expose + private int inserted; + @Expose + private int updated; + @Expose + private int skipped; + @Expose + private int conflicts; + @Expose + private int failedRecords; + /** why the table as a whole failed; null when the table itself was fine */ + @Expose + private String failureReason; + /** distinct record-level failure reasons, each with the number of records */ + @Expose + private Map recordFailureReasons = new LinkedHashMap<>(); + + public DownSyncTableResult(String groupName, String schemaName, String tableName) { + this.groupName = groupName; + this.schemaName = schemaName; + this.tableName = tableName; + this.status = STATUS_SUCCESS; + } + + /** the table could not be processed at all - nothing was delivered */ + public void tableFailed(String reason) { + this.status = STATUS_FAILED; + this.failureReason = reason; + } + + /** one record failed; the table itself is fine unless every record failed */ + public void recordFailed(String reason) { + this.failedRecords++; + String key = (reason == null || reason.trim().isEmpty()) ? "Unknown error" : reason.trim(); + recordFailureReasons.merge(key, 1, Integer::sum); + if (STATUS_SUCCESS.equals(this.status)) + this.status = STATUS_PARTIAL; + } + + public int getRecordsProcessed() { + return inserted + updated + skipped + conflicts + failedRecords; + } + + public void addInserted() { + inserted++; + } + + public void addUpdated() { + updated++; + } + + public void addSkipped() { + skipped++; + } + + public void addConflict() { + conflicts++; + } + + public String getGroupName() { + return groupName; + } + + public String getSchemaName() { + return schemaName; + } + + public String getTableName() { + return tableName; + } + + public String getStatus() { + return status; + } + + public int getInserted() { + return inserted; + } + + public int getUpdated() { + return updated; + } + + public int getSkipped() { + return skipped; + } + + public int getConflicts() { + return conflicts; + } + + public int getFailedRecords() { + return failedRecords; + } + + public String getFailureReason() { + return failureReason; + } + + public Map getRecordFailureReasons() { + return recordFailureReasons; + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 6bbc33f8..ab068adb 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -23,6 +23,7 @@ import java.sql.Timestamp; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -143,13 +144,33 @@ public int[] updateLatestMasterInLocal(String query, List syncDataList // ---------------------------------- Down-Sync Repository (central -> local) + public Map getSyncGroupNamesByTable() { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT LOWER(TRIM(s.TableName)) AS TableName, g.SyncTableGroupName " + + " FROM db_iemr.m_synctabledetail s " + + " JOIN db_iemr.m_synctablegroup g ON g.SyncTableGroupID = s.SyncTableGroupID " + + " WHERE IFNULL(s.Deleted, b'0') = b'0' AND IFNULL(g.Deleted, b'0') = b'0' " + + " AND s.TableName IS NOT NULL "; + + Map groups = new LinkedHashMap<>(); + for (Map row : jdbcTemplate.queryForList(query)) { + Object table = row.get("TableName"); + Object group = row.get("SyncTableGroupName"); + if (table != null && group != null) + groups.putIfAbsent(String.valueOf(table), String.valueOf(group)); + } + return groups; + } + public List getDownSyncColumns(String schema, String table) { jdbcTemplate = getJdbcTemplate(); String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + " AND COLUMN_NAME NOT IN ('DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " - + " 'LastDownSyncDate') " + " ORDER BY COLUMN_NAME "; + + " 'LastDownSyncDate', 'Processed', 'SyncFailureReason', 'SyncedBy', 'SyncedDate') " + + " ORDER BY COLUMN_NAME "; return jdbcTemplate.queryForList(query, String.class, schema, table); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 632af829..870bcd9c 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -25,6 +25,7 @@ import java.text.DateFormat; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; @@ -51,6 +52,7 @@ import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableResult; import com.iemr.mmu.repo.syncActivity_syncLayer.DownSyncTableDetailRepo; import com.iemr.mmu.utils.RestTemplateUtil; @@ -64,6 +66,7 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String VAN_SERIAL_NO = "VanSerialNo"; private static final String VAN_ID = "VanID"; private static final String LAST_MOD_DATE = "LastModDate"; + private static final String DATA_SYNC_CALL = "datasync"; private static final String LAST_DOWN_SYNC_DATE = "LastDownSyncDate"; private static final String DOWN_SYNCED = "DownSynced"; private static final String DOWN_SYNC_DATE = "DownSyncDate"; @@ -90,8 +93,12 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static int updatedCounter = 0; private static int conflictCounter = 0; private static int skippedCounter = 0; - private static int failedCounter = 0; + private static int failedTableCounter = 0; + private static int failedRecordCounter = 0; private static StringBuilder failedTables = new StringBuilder(); + private static final List tableResults = + Collections.synchronizedList(new ArrayList<>()); + private static DownSyncTableResult currentResult; private static String currentTable = ""; public String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, @@ -113,21 +120,35 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer updatedCounter = 0; conflictCounter = 0; skippedCounter = 0; - failedCounter = 0; + failedTableCounter = 0; + failedRecordCounter = 0; failedTables = new StringBuilder(); + tableResults.clear(); + currentResult = null; currentTable = ""; + final Map syncGroups = dataSyncRepository.getSyncGroupNamesByTable(); + final ExecutorService threadPool = Executors.newSingleThreadExecutor(); try { threadPool.submit(() -> { try { for (DownSyncTableDetail tableDetail : downSyncTables) { currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); + + String groupName = syncGroups.getOrDefault( + tableDetail.getTableName().toLowerCase(), + tableDetail.isMasterTable() ? "Masters" : "Other"); + currentResult = new DownSyncTableResult(groupName, tableDetail.getSchemaName(), + tableDetail.getTableName()); + tableResults.add(currentResult); + try { downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); } catch (Exception e) { - failedCounter++; + failedTableCounter++; failedTables.append(tableDetail.getTableName()).append(" | "); + currentResult.tableFailed(shorten(e.getMessage())); logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); } progressCounter++; @@ -151,8 +172,13 @@ public Map getDownSyncStatus() { resultMap.put("currentTable", currentTable); resultMap.put("totalTables", totalCounter); resultMap.put("completedTables", progressCounter); - resultMap.put("failedTableCount", failedCounter); + resultMap.put("successTableCount", progressCounter - failedTableCounter); + resultMap.put("failedTableCount", failedTableCounter); + resultMap.put("failedRecordCount", failedRecordCounter); resultMap.put("failedTables", failedTables.toString()); + synchronized (tableResults) { + resultMap.put("tableResults", new ArrayList<>(tableResults)); + } resultMap.put("recordsInserted", insertedCounter); resultMap.put("recordsUpdated", updatedCounter); resultMap.put("recordsSkipped", skippedCounter); @@ -215,7 +241,8 @@ private List> fetchDataFromCentral(DownSyncTableDetail table providerServiceMapID); RestTemplate restTemplate = new RestTemplate(); - HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, + DATA_SYNC_CALL); ResponseEntity response = restTemplate.exchange(downSyncDataUrl, HttpMethod.POST, request, String.class); if (response == null || !response.hasBody()) @@ -300,6 +327,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD if (localRecord == null) { Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); insertedCounter++; + if (currentResult != null) + currentResult.addInserted(); acks.add(DownSyncRecordAck.success(centralID, localID)); continue; } @@ -308,6 +337,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD if (!isCentralCopyNewer(record, localRecord, tableDetail.getLastModColumnName())) { skippedCounter++; + if (currentResult != null) + currentResult.addSkipped(); acks.add(DownSyncRecordAck.success(centralID, localID)); continue; } @@ -316,6 +347,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD dataSyncRepository.markDownSyncConflictInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, localID); conflictCounter++; + if (currentResult != null) + currentResult.addConflict(); acks.add(DownSyncRecordAck.failure(centralID, localID, DownSyncRecordAck.CONFLICT)); logger.warn("Down-sync conflict for {}.{}, local id {}", tableDetail.getSchemaName(), tableDetail.getTableName(), localID); @@ -324,13 +357,15 @@ private List saveTransactionalData(DownSyncTableDetail tableD updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, localID); updatedCounter++; + if (currentResult != null) + currentResult.addUpdated(); acks.add(DownSyncRecordAck.success(centralID, localID)); } catch (Exception e) { - failedCounter++; - String reason = e.getMessage() != null && e.getMessage().length() > 250 - ? e.getMessage().substring(0, 250) - : e.getMessage(); + failedRecordCounter++; + String reason = shorten(e.getMessage()); + if (currentResult != null) + currentResult.recordFailed(reason); acks.add(DownSyncRecordAck.failure(centralID, toLong(vanSerialNo), reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); @@ -381,6 +416,10 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo values.add(null); columns.add(LAST_DOWN_SYNC_DATE); values.add(Timestamp.valueOf(LocalDateTime.now())); + columns.add(DOWN_SYNCED); + values.add(DownSyncRecordAck.STATUS_PROCESSED); + columns.add(DOWN_SYNC_DATE); + values.add(Timestamp.valueOf(LocalDateTime.now())); StringBuilder placeHolders = new StringBuilder(); for (int i = 0; i < columns.size(); i++) { @@ -413,7 +452,9 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo } setClause.append(setClause.length() > 0 ? ", " : "").append(PROCESSED).append(" = 'P', ") - .append(SYNC_FAILURE_REASON).append(" = NULL, ").append(LAST_DOWN_SYNC_DATE).append(" = ? "); + .append(SYNC_FAILURE_REASON).append(" = NULL, ").append(LAST_DOWN_SYNC_DATE).append(" = ?, ") + .append(DOWN_SYNCED).append(" = 'P', ").append(DOWN_SYNC_DATE).append(" = ? "); + values.add(Timestamp.valueOf(LocalDateTime.now())); values.add(Timestamp.valueOf(LocalDateTime.now())); values.add(localID); @@ -423,6 +464,13 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); } + private static String shorten(String message) { + if (message == null || message.trim().isEmpty()) + return "Unknown error"; + String trimmed = message.trim(); + return trimmed.length() > 250 ? trimmed.substring(0, 250) : trimmed; + } + private void requireIdentifier(String value, String what, DownSyncTableDetail tableDetail) throws Exception { if (value == null || !value.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) throw new Exception("The down-sync configuration of " + tableDetail.getTableName() + " holds an invalid " @@ -444,7 +492,8 @@ private void acknowledgeToCentral(DownSyncTableDetail tableDetail, Integer vanID DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, acks); RestTemplate restTemplate = new RestTemplate(); - HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, + DATA_SYNC_CALL); ResponseEntity response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, String.class); From c8c5cb783cbce4e9223231ecbbee0154b67d2af3 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 15:40:44 +0530 Subject: [PATCH 18/81] fix: update the continuous call on check sync progress --- .../dataSyncActivity/StartSyncActivity.java | 7 +-- .../DownSyncDataFromServerImpl.java | 54 +++++++++---------- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index baf098da..b517479c 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -255,10 +255,11 @@ public String startDownSync(@RequestBody String requestOBJ, String s = downSyncDataFromServerImpl.startDownSync(serverAuthorization, jwtToken, obj.getInt("vanID"), providerServiceMapID); - if (s != null && s.equalsIgnoreCase("inProgress")) + if (s != null && s.equalsIgnoreCase("inProgress")) { response.setError(5000, "Down-sync is already in progress, kindly wait for it to finish"); - else - response.setResponse(s); + } else { + response.setResponse(new Gson().toJson(downSyncDataFromServerImpl.getDownSyncStatus())); + } } else { response.setError(5000, "vanID is missing, Kindly contact the administrator."); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 870bcd9c..e0a19be0 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -30,8 +30,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import org.json.JSONObject; @@ -129,40 +127,36 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer final Map syncGroups = dataSyncRepository.getSyncGroupNamesByTable(); - final ExecutorService threadPool = Executors.newSingleThreadExecutor(); try { - threadPool.submit(() -> { + for (DownSyncTableDetail tableDetail : downSyncTables) { + currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); + + String groupName = syncGroups.getOrDefault(tableDetail.getTableName().toLowerCase(), + tableDetail.isMasterTable() ? "Masters" : "Other"); + currentResult = new DownSyncTableResult(groupName, tableDetail.getSchemaName(), + tableDetail.getTableName()); + tableResults.add(currentResult); + try { - for (DownSyncTableDetail tableDetail : downSyncTables) { - currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); - - String groupName = syncGroups.getOrDefault( - tableDetail.getTableName().toLowerCase(), - tableDetail.isMasterTable() ? "Masters" : "Other"); - currentResult = new DownSyncTableResult(groupName, tableDetail.getSchemaName(), - tableDetail.getTableName()); - tableResults.add(currentResult); - - try { - downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); - } catch (Exception e) { - failedTableCounter++; - failedTables.append(tableDetail.getTableName()).append(" | "); - currentResult.tableFailed(shorten(e.getMessage())); - logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); - } - progressCounter++; - } - } finally { - IN_PROGRESS.set(false); + downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); + } catch (Exception e) { + failedTableCounter++; + failedTables.append(tableDetail.getTableName()).append(" | "); + currentResult.tableFailed(shorten(e.getMessage())); + logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); } - return "Down-sync completed"; - }); + progressCounter++; + } } finally { - threadPool.shutdown(); + IN_PROGRESS.set(false); + currentTable = ""; } - return " Down-sync started "; + logger.info("Down-sync finished : {} of {} tables succeeded, {} inserted, {} updated, {} skipped, " + + "{} conflicts, {} failed records", progressCounter - failedTableCounter, totalCounter, + insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter); + + return "Down-sync completed"; } public Map getDownSyncStatus() { From 2f354839db188f5743f149b82f05771afb20f5cf Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 16:22:43 +0530 Subject: [PATCH 19/81] fix: column name issue for stoptb tables --- .../service/dataSyncActivity/DataSyncRepository.java | 12 ++++++++++++ .../dataSyncActivity/DownSyncDataFromServerImpl.java | 12 +++++++++--- .../DataSyncRepositoryCentralDownload.java | 12 ++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index ab068adb..7f1d496d 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -163,6 +163,18 @@ public Map getSyncGroupNamesByTable() { return groups; } + public String resolveLastModColumn(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + + " AND COLUMN_NAME IN ('LastModDate', 'last_mod_date') " + + " ORDER BY FIELD(COLUMN_NAME, 'LastModDate', 'last_mod_date') "; + + List found = jdbcTemplate.queryForList(query, String.class, schema, table); + return (found == null || found.isEmpty()) ? null : found.get(0); + } + public List getDownSyncColumns(String schema, String table) { jdbcTemplate = getJdbcTemplate(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index e0a19be0..28bee1a8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -305,7 +305,13 @@ private List saveTransactionalData(DownSyncTableDetail tableD pkColumn = pkColumn.trim(); requireIdentifier(pkColumn, "VanAutoIncColumnName", tableDetail); - requireIdentifier(tableDetail.getLastModColumnName(), "LastModColumnName", tableDetail); + + String lastModColumn = dataSyncRepository.resolveLastModColumn(tableDetail.getSchemaName(), + tableDetail.getTableName()); + if (lastModColumn == null) + throw new Exception(tableDetail.getSchemaName() + "." + tableDetail.getTableName() + + " has neither LastModDate nor last_mod_date, so a change cannot be dated"); + requireIdentifier(lastModColumn, "modification-time column", tableDetail); List acks = new ArrayList<>(); @@ -316,7 +322,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD try { Map localRecord = dataSyncRepository.getLocalRecordForDownSync( tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, vanSerialNo, vanID, - tableDetail.getLastModColumnName()); + lastModColumn); if (localRecord == null) { Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); @@ -329,7 +335,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD Long localID = toLong(localRecord.get(pkColumn)); - if (!isCentralCopyNewer(record, localRecord, tableDetail.getLastModColumnName())) { + if (!isCentralCopyNewer(record, localRecord, lastModColumn)) { skippedCounter++; if (currentResult != null) currentResult.addSkipped(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 2cceb6d8..aa2db2a0 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -212,6 +212,18 @@ else if (masterType.equalsIgnoreCase("P")) { // ---------------------------------- Down-Sync Repository (central -> local) + public String resolveLastModColumn(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + + " AND COLUMN_NAME IN ('LastModDate', 'last_mod_date') " + + " ORDER BY FIELD(COLUMN_NAME, 'LastModDate', 'last_mod_date') "; + + List found = jdbcTemplate.queryForList(query, String.class, schema, table); + return (found == null || found.isEmpty()) ? null : found.get(0); + } + public List> getDownSyncDataFromTable(String schema, String table, String columnNames, String tableType, Integer vanID, String lastModColumn) throws Exception { jdbcTemplate = getJdbcTemplate(); From 7b2213b671dff182865dd46dc2e32c3afe0c924c Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 16:46:58 +0530 Subject: [PATCH 20/81] fix: issue in stoptb tables --- .../GetDownSyncDataFromCentralImpl.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java index 54b931ac..11d0bac2 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -63,24 +63,22 @@ public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) t List> resultSetList = dataSyncRepositoryCentralDownload.getDownSyncDataFromTable( downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), - downSyncDataDigester.getVanID(), resolveLastModColumn(downSyncDataDigester.getTableName())); + downSyncDataDigester.getVanID(), + resolveLastModColumn(downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName())); return downSyncGson().toJson(resultSetList); } - private String resolveLastModColumn(String tableName) throws Exception { - String lastModColumn = "LastModDate"; + private String resolveLastModColumn(String schemaName, String tableName) throws Exception { + String lastModColumn = dataSyncRepositoryCentralDownload.resolveLastModColumn(schemaName, tableName); - if (tableName != null) { - ArrayList tableDetails = downSyncTableDetailRepo - .getActiveDownSyncTableByName(tableName.trim()); - if (tableDetails != null && !tableDetails.isEmpty()) - lastModColumn = tableDetails.get(0).getLastModColumnName(); - } + if (lastModColumn == null) + throw new Exception(schemaName + "." + tableName + + " has neither LastModDate nor last_mod_date, so the down-sync cannot tell when a record was" + + " last changed"); - if (lastModColumn == null || !lastModColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) - throw new Exception( - "The down-sync configuration of " + tableName + " holds an invalid modification-time column name"); + if (!lastModColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + throw new Exception(schemaName + "." + tableName + " holds an invalid modification-time column name"); return lastModColumn; } From a9e7768c295e850ac3825876abe99381752b1196 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 11:19:14 +0530 Subject: [PATCH 21/81] fix: add central id --- .../dataSyncActivity/DataSyncRepository.java | 16 +++++++++---- .../DownSyncDataFromServerImpl.java | 23 +++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 7f1d496d..2ce57c7a 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -180,7 +180,7 @@ public List getDownSyncColumns(String schema, String table) { String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " - + " AND COLUMN_NAME NOT IN ('DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " + + " AND COLUMN_NAME NOT IN ('CentralID', 'DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " + " 'LastDownSyncDate', 'Processed', 'SyncFailureReason', 'SyncedBy', 'SyncedDate') " + " ORDER BY COLUMN_NAME "; @@ -188,16 +188,16 @@ public List getDownSyncColumns(String schema, String table) { } public Map getLocalRecordForDownSync(String schema, String table, String autoIncColumnName, - Object vanSerialNo, Object vanID, String lastModColumn) { - if (vanSerialNo == null) + Object centralID, Object vanID, String lastModColumn) { + if (centralID == null) return null; jdbcTemplate = getJdbcTemplate(); String query = " SELECT " + autoIncColumnName + ", Processed, " + lastModColumn + " AS LastModDate, LastDownSyncDate FROM " + schema + "." + table - + " WHERE " + autoIncColumnName + " = ? AND VanID = ? "; + + " WHERE CentralID = ? AND VanID = ? "; - List> resultSet = jdbcTemplate.queryForList(query, vanSerialNo, vanID); + List> resultSet = jdbcTemplate.queryForList(query, centralID, vanID); if (resultSet == null || resultSet.isEmpty()) return null; @@ -225,6 +225,12 @@ public int updateDownSyncRecordInLocal(String query, Object[] params) { return jdbcTemplate.update(query, params); } + public int updateVanSerialNoInLocal(String schema, String table, String autoIncColumnName, Object localID) { + jdbcTemplate = getJdbcTemplate(); + String query = " UPDATE " + schema + "." + table + " SET VanSerialNo = ? WHERE " + autoIncColumnName + " = ? "; + return jdbcTemplate.update(query, localID, localID); + } + public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); String query = " UPDATE " + schema + "." + table diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 28bee1a8..171051bd 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -71,6 +71,7 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String DOWN_SYNC_FAILURE_REASON = "DownSyncFailureReason"; private static final String PROCESSED = "Processed"; private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; + private static final String CENTRAL_ID = "CentralID"; @Value("${downSyncDataUrl}") private String downSyncDataUrl; @@ -317,15 +318,14 @@ private List saveTransactionalData(DownSyncTableDetail tableD for (Map record : dataFromCentral) { Long centralID = toLong(record.get(pkColumn)); - Object vanSerialNo = normalize(record.get(VAN_SERIAL_NO)); try { Map localRecord = dataSyncRepository.getLocalRecordForDownSync( - tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, vanSerialNo, vanID, + tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, centralID, vanID, lastModColumn); if (localRecord == null) { - Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); + Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, centralID); insertedCounter++; if (currentResult != null) currentResult.addInserted(); @@ -366,7 +366,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD String reason = shorten(e.getMessage()); if (currentResult != null) currentResult.recordFailed(reason); - acks.add(DownSyncRecordAck.failure(centralID, toLong(vanSerialNo), reason)); + acks.add(DownSyncRecordAck.failure(centralID, null, reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); } @@ -395,7 +395,7 @@ private boolean hasUnsentLocalChanges(Map localRecord) { } private Long insertRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, - String pkColumn, Map record) { + String pkColumn, Map record, Long centralID) { List columns = new ArrayList<>(); List values = new ArrayList<>(); @@ -410,6 +410,8 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo values.add(normalize(record.get(serverColumns.get(i)))); } + columns.add(CENTRAL_ID); + values.add(centralID); columns.add(PROCESSED); values.add("P"); columns.add(SYNC_FAILURE_REASON); @@ -431,7 +433,13 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo String query = " INSERT INTO " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " ( " + String.join(", ", columns) + " ) VALUES ( " + placeHolders + " ) "; - return dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + Long localID = dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + + if (localID != null) + dataSyncRepository.updateVanSerialNoInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), + pkColumn, localID); + + return localID; } private void updateRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, @@ -478,7 +486,8 @@ private void requireIdentifier(String value, String what, DownSyncTableDetail ta } private boolean isDownSyncManagedColumn(String column) { - return PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) + return VAN_SERIAL_NO.equalsIgnoreCase(column) || CENTRAL_ID.equalsIgnoreCase(column) + || PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) || LAST_DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNCED.equalsIgnoreCase(column) || DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNC_FAILURE_REASON.equalsIgnoreCase(column); } From c01a97b82615fa3bbf6b5c9532765bed7b0aae94 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 12:43:41 +0530 Subject: [PATCH 22/81] fix: FK Mapping issue --- .../DownSyncTableDetail.java | 14 +- .../dataSyncActivity/DataSyncRepository.java | 27 +++ .../DownSyncDataFromServerImpl.java | 199 ++++++++++++++++-- 3 files changed, 221 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index c7a0dd68..9ffadb84 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -70,8 +70,12 @@ public class DownSyncTableDetail { @Expose @Column(name = "TableType") private String tableType; + + @Expose + @Column(name = "FkColumnMapping") + private String fkColumnMapping; - /** lower number syncs first - enforces the FK dependency chain */ + @Expose @Column(name = "SyncOrder") private Integer syncOrder; @@ -149,6 +153,14 @@ public void setVanAutoIncColumnName(String vanAutoIncColumnName) { this.vanAutoIncColumnName = vanAutoIncColumnName; } + public String getFkColumnMapping() { + return fkColumnMapping; + } + + public void setFkColumnMapping(String fkColumnMapping) { + this.fkColumnMapping = fkColumnMapping; + } + public String getTableType() { return tableType; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 2ce57c7a..66ddfc03 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -231,6 +231,33 @@ public int updateVanSerialNoInLocal(String schema, String table, String autoIncC return jdbcTemplate.update(query, localID, localID); } + + public Long resolveLocalIdForCentralValue(String schema, String table, String pkColumn, Object centralValue, + Object vanID) { + if (centralValue == null) + return null; + + jdbcTemplate = getJdbcTemplate(); + String from = " FROM " + schema + "." + table + " WHERE "; + + Long resolved = queryForFirstLong(" SELECT " + pkColumn + from + " CentralID = ? AND VanID = ? ", centralValue, + vanID); + if (resolved != null) + return resolved; + + resolved = queryForFirstLong(" SELECT " + pkColumn + from + " VanSerialNo = ? AND VanID = ? ", centralValue, + vanID); + if (resolved != null) + return resolved; + + return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); + } + + private Long queryForFirstLong(String query, Object... params) { + List found = jdbcTemplate.queryForList(query, Long.class, params); + return (found == null || found.isEmpty()) ? null : found.get(0); + } + public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); String query = " UPDATE " + schema + "." + table diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 171051bd..b8482f1f 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -72,6 +72,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String PROCESSED = "Processed"; private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; private static final String CENTRAL_ID = "CentralID"; + + private static final int ACK_BATCH_SIZE = 500; @Value("${downSyncDataUrl}") private String downSyncDataUrl; @@ -314,18 +316,41 @@ private List saveTransactionalData(DownSyncTableDetail tableD + " has neither LastModDate nor last_mod_date, so a change cannot be dated"); requireIdentifier(lastModColumn, "modification-time column", tableDetail); + Map fkMapping = parseFkColumnMapping(tableDetail); + Map fkCache = new LinkedHashMap<>(); + List acks = new ArrayList<>(); - for (Map record : dataFromCentral) { - Long centralID = toLong(record.get(pkColumn)); + for (Map incoming : dataFromCentral) { + Map record = incoming; + // Resolved ignoring case : column names differ in case between tables + // (BenConsentID vs BenConsentId, vanSerialNo vs VanSerialNo) and a map + // lookup is case-sensitive, so taking the configured spelling literally + // yields null and every record then looks new on every run. + Long centralID = toLong(record.get(resolveKeyIgnoringCase(record, pkColumn))); try { + if (centralID == null) + throw new Exception("Central sent no value for the primary key '" + pkColumn + + "'. Check VanAutoIncColumnName in m_downsynctabledetail against the real column of " + + tableDetail.getTableName() + " - writing the record without it inserts a duplicate on" + + " every later run."); + Map localRecord = dataSyncRepository.getLocalRecordForDownSync( tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, centralID, vanID, lastModColumn); + // a pointer at another table's primary key means a different number here + // than at central, so it is rewritten before the row is written + StringBuilder unresolved = new StringBuilder(); + Map mapped = translateForeignKeys(fkMapping, record, vanID, fkCache, unresolved); + if (localRecord == null) { - Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, centralID); + if (mapped == null) + throw new Exception("Cannot place this record : " + shorten(unresolved.toString()) + + " Its parent row has not reached this van yet."); + + Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, mapped, centralID); insertedCounter++; if (currentResult != null) currentResult.addInserted(); @@ -355,7 +380,10 @@ private List saveTransactionalData(DownSyncTableDetail tableD continue; } - updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, localID); + // on an update an unresolvable pointer is left at its current local value + // rather than being overwritten with an id that means nothing here + updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, mapped != null ? mapped : record, + localID, mapped == null ? fkMapping.keySet() : null); updatedCounter++; if (currentResult != null) currentResult.addUpdated(); @@ -443,7 +471,7 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo } private void updateRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, - String pkColumn, Map record, Long localID) { + String pkColumn, Map record, Long localID, java.util.Set columnsToLeaveAlone) { StringBuilder setClause = new StringBuilder(); List values = new ArrayList<>(); @@ -452,6 +480,8 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo String vanColumn = vanColumns.get(i); if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) continue; + if (columnsToLeaveAlone != null && containsIgnoringCase(columnsToLeaveAlone, vanColumn)) + continue; if (setClause.length() > 0) setClause.append(", "); @@ -472,6 +502,121 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); } + /*** + * Parses FkColumnMapping - "childColumn:schema.parentTable", several separated + * by ';' - into an ordered map. Blank config gives an empty map, which is the + * "nothing to translate" case for almost every table. + */ + private Map parseFkColumnMapping(DownSyncTableDetail tableDetail) throws Exception { + Map mapping = new LinkedHashMap<>(); + String config = tableDetail.getFkColumnMapping(); + if (config == null || config.trim().isEmpty()) + return mapping; + + for (String entry : config.split(";")) { + if (entry == null || entry.trim().isEmpty()) + continue; + + String[] parts = entry.trim().split(":"); + if (parts.length != 2 || parts[1].trim().split("\\.").length != 2) + throw new Exception("FkColumnMapping of " + tableDetail.getTableName() + " holds '" + entry.trim() + + "', which is not of the form childColumn:schema.parentTable"); + + requireIdentifier(parts[0].trim(), "FkColumnMapping child column", tableDetail); + mapping.put(parts[0].trim(), parts[1].trim()); + } + return mapping; + } + + + private Map translateForeignKeys(Map fkMapping, Map record, + Integer vanID, Map cache, StringBuilder unresolved) { + + if (fkMapping.isEmpty()) + return record; + + Map translated = new LinkedHashMap<>(record); + + for (Map.Entry fk : fkMapping.entrySet()) { + String childColumn = resolveKeyIgnoringCase(record, fk.getKey()); + if (childColumn == null) + continue; + + Long centralValue = toLong(record.get(childColumn)); + if (centralValue == null) + continue; + + String parent = fk.getValue(); + String cacheKey = parent + "#" + centralValue; + Long localValue = cache.get(cacheKey); + + if (localValue == null) { + String parentSchema = parent.substring(0, parent.indexOf('.')); + String parentTable = parent.substring(parent.indexOf('.') + 1); + String parentPK = parentPrimaryKey(parentSchema, parentTable); + + if (parentPK == null) { + unresolved.append(fk.getKey()).append(" -> ").append(parent) + .append(" (no VanAutoIncColumnName configured for the parent); "); + continue; + } + + localValue = dataSyncRepository.resolveLocalIdForCentralValue(parentSchema, parentTable, parentPK, + centralValue, vanID); + if (localValue != null) + cache.put(cacheKey, localValue); + } + + if (localValue == null) { + unresolved.append(fk.getKey()).append('=').append(centralValue).append(" not found in ").append(parent) + .append("; "); + continue; + } + + translated.put(childColumn, localValue); + } + + return unresolved.length() > 0 ? null : translated; + } + + /*** + * The parent's local primary key, taken from its own down-sync configuration. + */ + private String parentPrimaryKey(String parentSchema, String parentTable) { + ArrayList parents = downSyncTableDetailRepo.getActiveDownSyncTableByName(parentTable); + for (DownSyncTableDetail parent : parents) { + if (parentSchema.equalsIgnoreCase(parent.getSchemaName()) && parent.getVanAutoIncColumnName() != null + && parent.getVanAutoIncColumnName().trim().matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + return parent.getVanAutoIncColumnName().trim(); + } + return null; + } + + /*** + * Column names differ in case between the tables (vanSerialNo vs VanSerialNo), + * and a map lookup is case-sensitive - so never index a result row by a + * hard-coded name without this. + */ + private String resolveKeyIgnoringCase(Map record, String column) { + if (record == null || column == null) + return null; + if (record.containsKey(column)) + return column; + for (String key : record.keySet()) { + if (key.equalsIgnoreCase(column)) + return key; + } + return null; + } + + private boolean containsIgnoringCase(java.util.Set columns, String column) { + for (String candidate : columns) { + if (candidate.equalsIgnoreCase(column)) + return true; + } + return false; + } + private static String shorten(String message) { if (message == null || message.trim().isEmpty()) return "Unknown error"; @@ -498,23 +643,41 @@ private void acknowledgeToCentral(DownSyncTableDetail tableDetail, Integer vanID if (acks == null || acks.isEmpty()) return; - DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, acks); - RestTemplate restTemplate = new RestTemplate(); - HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, - DATA_SYNC_CALL); - ResponseEntity response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, - String.class); + int acknowledged = 0; - if (response == null || !response.hasBody()) - throw new Exception("Empty response while updating the down-sync flag for " + tableDetail.getTableName()); + for (int from = 0; from < acks.size(); from += ACK_BATCH_SIZE) { + List batch = acks.subList(from, Math.min(from + ACK_BATCH_SIZE, acks.size())); - JSONObject responseObj = new JSONObject(response.getBody()); - if (!responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) - throw new Exception("Could not update the down-sync flag at central for " + tableDetail.getTableName() - + " : " + responseObj.optString("errorMessage")); + DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, batch); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, + DATA_SYNC_CALL); + + ResponseEntity response; + try { + response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, String.class); + } catch (Exception e) { + // the batches already accepted stay acknowledged - say how far we got, so + // the gap between the two sides is visible rather than guessed at + throw new Exception("Down-sync flag update failed for " + tableDetail.getTableName() + " after " + + acknowledged + " of " + acks.size() + " records : " + e.getMessage(), e); + } + + if (response == null || !response.hasBody()) + throw new Exception("Empty response while updating the down-sync flag for " + + tableDetail.getTableName() + " after " + acknowledged + " of " + acks.size() + " records"); + + JSONObject responseObj = new JSONObject(response.getBody()); + if (!responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) + throw new Exception("Could not update the down-sync flag at central for " + + tableDetail.getTableName() + " after " + acknowledged + " of " + acks.size() + " records : " + + responseObj.optString("errorMessage")); + + acknowledged += batch.size(); + } - logger.info("Down-sync flag updated at central for {} records of {}", acks.size(), tableDetail.getTableName()); + logger.info("Down-sync flag updated at central for {} records of {} in {} batch(es)", acknowledged, + tableDetail.getTableName(), (acks.size() + ACK_BATCH_SIZE - 1) / ACK_BATCH_SIZE); } private List splitColumns(String columns) { From 6dfae6173836eca13b8b56318ac9989d628d1815 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 12:55:19 +0530 Subject: [PATCH 23/81] fix: downsync issue --- .../DownSyncDataDigester.java | 22 +++++- .../DownSyncDataFromServerImpl.java | 68 ++++++++++++++++--- .../DataSyncRepositoryCentralDownload.java | 29 ++++++-- .../GetDownSyncDataFromCentralImpl.java | 4 +- 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java index af143161..c250cf73 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java @@ -47,12 +47,14 @@ public class DownSyncDataDigester { /** populated only on the acknowledgement call */ @Expose private List records; + private Long lastFetchedID; + private Integer batchSize; public DownSyncDataDigester() { } public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, String serverColumnName, - Integer vanID, Integer providerServiceMapID) { + Integer vanID, Integer providerServiceMapID, Long lastFetchedID, Integer batchSize) { DownSyncDataDigester digester = new DownSyncDataDigester(); digester.schemaName = tableDetail.getSchemaName(); digester.tableName = tableDetail.getTableName(); @@ -61,6 +63,8 @@ public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, digester.tableType = tableDetail.getTableType(); digester.vanID = vanID; digester.providerServiceMapID = providerServiceMapID; + digester.lastFetchedID = lastFetchedID; + digester.batchSize = batchSize; return digester; } @@ -132,6 +136,22 @@ public void setProviderServiceMapID(Integer providerServiceMapID) { this.providerServiceMapID = providerServiceMapID; } + public Long getLastFetchedID() { + return lastFetchedID; + } + + public void setLastFetchedID(Long lastFetchedID) { + this.lastFetchedID = lastFetchedID; + } + + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + public List getRecords() { return records; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index b8482f1f..7e1396de 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -74,6 +74,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String CENTRAL_ID = "CentralID"; private static final int ACK_BATCH_SIZE = 500; + + private static final int FETCH_BATCH_SIZE = 1000; @Value("${downSyncDataUrl}") private String downSyncDataUrl; @@ -213,29 +215,75 @@ private void downSyncTable(DownSyncTableDetail tableDetail, Integer vanID, Integ + tableDetail.getTableName() + " (" + serverColumns.size() + " vs " + vanColumns.size() + "). Central and local must expose the same number of columns, in the same order."); - List> dataFromCentral = fetchDataFromCentral(tableDetail, String.join(",", serverColumns), - vanID, providerServiceMapID, serverAuthorization, jwtToken); + String columnList = String.join(",", serverColumns); - if (dataFromCentral.isEmpty()) { - logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), tableDetail.getTableName()); + if (tableDetail.isMasterTable()) { + // masters are a full pull with no primary key to page on, and are small + List> dataFromCentral = fetchDataFromCentral(tableDetail, columnList, vanID, + providerServiceMapID, serverAuthorization, jwtToken, null, null); + + if (dataFromCentral.isEmpty()) { + logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), + tableDetail.getTableName()); + return; + } + upsertMasterData(tableDetail, serverColumns, vanColumns, dataFromCentral); return; } - if (tableDetail.isMasterTable()) { - upsertMasterData(tableDetail, serverColumns, vanColumns, dataFromCentral); - } else { + String pkColumn = tableDetail.getVanAutoIncColumnName(); + requireIdentifier(pkColumn == null ? null : pkColumn.trim(), "VanAutoIncColumnName", tableDetail); + + Long lastFetchedID = null; + int page = 0; + int totalFetched = 0; + + while (true) { + List> dataFromCentral = fetchDataFromCentral(tableDetail, columnList, vanID, + providerServiceMapID, serverAuthorization, jwtToken, lastFetchedID, FETCH_BATCH_SIZE); + + if (dataFromCentral.isEmpty()) + break; + + page++; + totalFetched += dataFromCentral.size(); + List acks = saveTransactionalData(tableDetail, serverColumns, vanColumns, dataFromCentral, vanID); acknowledgeToCentral(tableDetail, vanID, acks, serverAuthorization, jwtToken); + + Long highestID = highestCentralID(dataFromCentral, pkColumn.trim()); + if (highestID == null || (lastFetchedID != null && highestID <= lastFetchedID)) + break; + lastFetchedID = highestID; + + if (dataFromCentral.size() < FETCH_BATCH_SIZE) + break; + } + + if (totalFetched == 0) + logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), tableDetail.getTableName()); + else + logger.info("Down-synced {} records of {}.{} in {} page(s)", totalFetched, tableDetail.getSchemaName(), + tableDetail.getTableName(), page); + } + + private Long highestCentralID(List> records, String pkColumn) { + Long highest = null; + for (Map record : records) { + Long id = toLong(record.get(resolveKeyIgnoringCase(record, pkColumn))); + if (id != null && (highest == null || id > highest)) + highest = id; } + return highest; } private List> fetchDataFromCentral(DownSyncTableDetail tableDetail, String serverColumnName, - Integer vanID, Integer providerServiceMapID, String serverAuthorization, String jwtToken) - throws Exception { + Integer vanID, Integer providerServiceMapID, String serverAuthorization, String jwtToken, + Long lastFetchedID, Integer batchSize) throws Exception { DownSyncDataDigester digester = DownSyncDataDigester.forDownload(tableDetail, serverColumnName, vanID, - providerServiceMapID); + providerServiceMapID, lastFetchedID, batchSize); RestTemplate restTemplate = new RestTemplate(); HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index aa2db2a0..f9c2c557 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -225,7 +225,8 @@ public String resolveLastModColumn(String schema, String table) { } public List> getDownSyncDataFromTable(String schema, String table, String columnNames, - String tableType, Integer vanID, String lastModColumn) throws Exception { + String tableType, Integer vanID, String lastModColumn, String pkColumn, Long lastFetchedID, + Integer batchSize) throws Exception { jdbcTemplate = getJdbcTemplate(); if (schema == null || table == null) @@ -238,12 +239,28 @@ public List> getDownSyncDataFromTable(String schema, String if (vanID == null) throw new Exception("Invalid down-sync request. VanID is mandatory for transactional table " + table); - String query = " SELECT " + columns + " FROM " + schema + "." + table - + " WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') " - + " OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND " + lastModColumn - + " > DownSyncDate ) ) "; + boolean paged = pkColumn != null && pkColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$") && batchSize != null + && batchSize > 0; + + StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(schema) + .append(".").append(table) + .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') ") + .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(lastModColumn) + .append(" > DownSyncDate ) ) "); + + List params = new ArrayList<>(); + params.add(vanID); + + if (paged) { + if (lastFetchedID != null) { + query.append(" AND ").append(pkColumn).append(" > ? "); + params.add(lastFetchedID); + } + query.append(" ORDER BY ").append(pkColumn).append(" LIMIT ").append(batchSize.intValue()); + } + logger.info("Down-sync select query for {}.{} : {}", schema, table, query); - resultSetList = jdbcTemplate.queryForList(query, vanID); + resultSetList = jdbcTemplate.queryForList(query.toString(), params.toArray()); } else { String query = " SELECT " + columns + " FROM " + schema + "." + table; logger.info("Down-sync select query for {}.{} : {}", schema, table, query); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java index 11d0bac2..8dc1015f 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -64,7 +64,9 @@ public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) t downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), downSyncDataDigester.getVanID(), - resolveLastModColumn(downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName())); + resolveLastModColumn(downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName()), + downSyncDataDigester.getVanAutoIncColumnName(), downSyncDataDigester.getLastFetchedID(), + downSyncDataDigester.getBatchSize()); return downSyncGson().toJson(resultSetList); } From 9e25ef6f2ac478f12ae2931f903a34d23f86205a Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 13:24:55 +0530 Subject: [PATCH 24/81] fix: FK issue in i_ beneficiarymapping table --- .../DownSyncTableResult.java | 9 ++++++++ .../dataSyncActivity/DataSyncRepository.java | 9 ++++++++ .../DownSyncDataFromServerImpl.java | 22 ++++++++++++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java index 7c54b79d..8d253f05 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java @@ -70,6 +70,7 @@ public class DownSyncTableResult { /** distinct record-level failure reasons, each with the number of records */ @Expose private Map recordFailureReasons = new LinkedHashMap<>(); + private Map recordWarnings = new LinkedHashMap<>(); public DownSyncTableResult(String groupName, String schemaName, String tableName) { this.groupName = groupName; @@ -93,6 +94,14 @@ public void recordFailed(String reason) { this.status = STATUS_PARTIAL; } + public void recordWarning(String reason) { + recordWarnings.merge(reason, 1, Integer::sum); + } + + public Map getRecordWarnings() { + return recordWarnings; + } + public int getRecordsProcessed() { return inserted + updated + skipped + conflicts + failedRecords; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 66ddfc03..524ab899 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -253,6 +253,15 @@ public Long resolveLocalIdForCentralValue(String schema, String table, String pk return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); } + public boolean isColumnNullable(String schema, String table, String column) { + jdbcTemplate = getJdbcTemplate(); + List nullable = jdbcTemplate.queryForList( + " SELECT IS_NULLABLE FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ? ", + String.class, schema, table, column); + return !nullable.isEmpty() && "YES".equalsIgnoreCase(nullable.get(0)); + } + private Long queryForFirstLong(String query, Object... params) { List found = jdbcTemplate.queryForList(query, Long.class, params); return (found == null || found.isEmpty()) ? null : found.get(0); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 7e1396de..81b537a5 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -391,7 +391,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD // a pointer at another table's primary key means a different number here // than at central, so it is rewritten before the row is written StringBuilder unresolved = new StringBuilder(); - Map mapped = translateForeignKeys(fkMapping, record, vanID, fkCache, unresolved); + Map mapped = translateForeignKeys(tableDetail, fkMapping, record, vanID, fkCache, + unresolved); if (localRecord == null) { if (mapped == null) @@ -577,8 +578,8 @@ private Map parseFkColumnMapping(DownSyncTableDetail tableDetail } - private Map translateForeignKeys(Map fkMapping, Map record, - Integer vanID, Map cache, StringBuilder unresolved) { + private Map translateForeignKeys(DownSyncTableDetail tableDetail, Map fkMapping, + Map record, Integer vanID, Map cache, StringBuilder unresolved) { if (fkMapping.isEmpty()) return record; @@ -616,6 +617,21 @@ private Map translateForeignKeys(Map fkMapping, } if (localValue == null) { + // A pointer the van cannot place is only fatal if the column demands a + // value. Where it is optional - a consent or an image the van has no copy + // of - the record is worth more than the pointer, so it lands with the + // column empty and the loss is reported rather than hidden. + if (dataSyncRepository.isColumnNullable(tableDetail.getSchemaName(), tableDetail.getTableName(), + childColumn)) { + translated.put(childColumn, null); + String warning = fk.getKey() + " could not be placed (" + parent + + " has no such row on this van) and was left empty"; + if (currentResult != null) + currentResult.recordWarning(warning); + logger.warn("{}.{} : {}", tableDetail.getSchemaName(), tableDetail.getTableName(), warning); + continue; + } + unresolved.append(fk.getKey()).append('=').append(centralValue).append(" not found in ").append(parent) .append("; "); continue; From 898806257996c91bf55108485b9ba559c16f7875 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 14:04:46 +0530 Subject: [PATCH 25/81] fix: include U for downsync again --- .../data/syncActivity_syncLayer/DownSyncRecordAck.java | 10 ++++++++++ .../dataSyncActivity/DownSyncDataFromServerImpl.java | 2 +- .../DataSyncRepositoryCentralDownload.java | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java index 5be35f81..e0de23c2 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java @@ -27,6 +27,7 @@ public class DownSyncRecordAck { public static final String STATUS_PROCESSED = "P"; public static final String STATUS_FAILED = "F"; + public static final String STATUS_RETRY = "U"; public static final String CONFLICT = "CONFLICT"; @Expose @@ -56,6 +57,15 @@ public static DownSyncRecordAck failure(Long centralID, Long vanSerialNo, String return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_FAILED, failureReason); } + + public static DownSyncRecordAck retryable(Long centralID, Long vanSerialNo, String failureReason) { + return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_RETRY, failureReason); + } + + public boolean isRetryable() { + return STATUS_RETRY.equalsIgnoreCase(status); + } + public boolean isSuccess() { return STATUS_PROCESSED.equalsIgnoreCase(status); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 81b537a5..0fd0c509 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -443,7 +443,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD String reason = shorten(e.getMessage()); if (currentResult != null) currentResult.recordFailed(reason); - acks.add(DownSyncRecordAck.failure(centralID, null, reason)); + acks.add(DownSyncRecordAck.retryable(centralID, null, reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index f9c2c557..38005752 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -283,6 +283,7 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk List successWithSerialNo = new ArrayList<>(); List successWithoutSerialNo = new ArrayList<>(); List failed = new ArrayList<>(); + List retryable = new ArrayList<>(); for (DownSyncRecordAck record : records) { if (record == null || record.getCentralID() == null) @@ -293,6 +294,8 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk successWithSerialNo.add(new Object[] { record.getVanSerialNo(), record.getCentralID() }); else successWithoutSerialNo.add(new Object[] { record.getCentralID() }); + } else if (record.isRetryable()) { + retryable.add(new Object[] { record.getFailureReason(), record.getCentralID() }); } else { failed.add(new Object[] { record.getFailureReason(), record.getCentralID() }); } @@ -314,6 +317,12 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); } + if (!retryable.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, retryable)); + } + if (!failed.isEmpty()) { String query = " UPDATE " + schema + "." + table + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; From 7c68c8b15f5ea7259255a4a78c98a271eafa815d Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 14:51:40 +0530 Subject: [PATCH 26/81] fix: revert central id for m_beneficiaryregidmapping --- .../syncActivity_syncLayer/DownSyncTableDetail.java | 12 ++++++++++++ .../dataSyncActivity/DownSyncDataFromServerImpl.java | 11 +++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index 9ffadb84..1e22fdb6 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -74,6 +74,10 @@ public class DownSyncTableDetail { @Expose @Column(name = "FkColumnMapping") private String fkColumnMapping; + + @Expose + @Column(name = "PreserveCentralPK") + private Boolean preserveCentralPK; @Expose @@ -153,6 +157,14 @@ public void setVanAutoIncColumnName(String vanAutoIncColumnName) { this.vanAutoIncColumnName = vanAutoIncColumnName; } + public boolean isPreserveCentralPK() { + return Boolean.TRUE.equals(preserveCentralPK); + } + + public void setPreserveCentralPK(Boolean preserveCentralPK) { + this.preserveCentralPK = preserveCentralPK; + } + public String getFkColumnMapping() { return fkColumnMapping; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 0fd0c509..87aed76e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -477,10 +477,11 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo List columns = new ArrayList<>(); List values = new ArrayList<>(); + boolean preservePK = tableDetail.isPreserveCentralPK(); + for (int i = 0; i < vanColumns.size(); i++) { String vanColumn = vanColumns.get(i); - // the local auto-increment PK is skipped so that local generates its own value - if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + if ((vanColumn.equalsIgnoreCase(pkColumn) && !preservePK) || isDownSyncManagedColumn(vanColumn)) continue; columns.add(vanColumn); @@ -512,6 +513,12 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo Long localID = dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + if (preservePK && localID == null) + localID = centralID; + + if (localID == null && preservePK) + localID = centralID; + if (localID != null) dataSyncRepository.updateVanSerialNoInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, localID); From 822a04da33b705c91f7e8f7260c678fecc6c7dac Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 15:20:46 +0530 Subject: [PATCH 27/81] fix: conflict section --- .../DownSyncTableResult.java | 3 +++ .../dataSyncActivity/DataSyncRepository.java | 3 ++- .../DownSyncDataFromServerImpl.java | 23 ++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java index 8d253f05..a9691fbb 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java @@ -45,6 +45,7 @@ public class DownSyncTableResult { public static final String STATUS_SUCCESS = "SUCCESS"; public static final String STATUS_FAILED = "FAILED"; public static final String STATUS_PARTIAL = "PARTIAL"; + public static final String STATUS_CONFLICT = "CONFLICT"; @Expose private String groupName; @@ -120,6 +121,8 @@ public void addSkipped() { public void addConflict() { conflicts++; + if (STATUS_SUCCESS.equals(this.status)) + this.status = STATUS_CONFLICT; } public String getGroupName() { diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 524ab899..97e7fcd5 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -270,7 +270,8 @@ private Long queryForFirstLong(String query, Object... params) { public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); String query = " UPDATE " + schema + "." + table - + " SET Processed = 'F', SyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; + + " SET Processed = 'F', SyncFailureReason = 'CONFLICT'," + + " DownSynced = 'F', DownSyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; return jdbcTemplate.update(query, localID); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 87aed76e..963d4526 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -161,7 +161,8 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer + "{} conflicts, {} failed records", progressCounter - failedTableCounter, totalCounter, insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter); - return "Down-sync completed"; + return conflictCounter > 0 ? "Down-sync completed with " + conflictCounter + " conflict(s) to review" + : "Down-sync completed"; } public Map getDownSyncStatus() { @@ -182,9 +183,29 @@ public Map getDownSyncStatus() { resultMap.put("recordsUpdated", updatedCounter); resultMap.put("recordsSkipped", skippedCounter); resultMap.put("conflicts", conflictCounter); + resultMap.put("conflictsPending", conflictCounter > 0); + resultMap.put("summary", buildSummary()); return resultMap; } + private String buildSummary() { + StringBuilder summary = new StringBuilder(); + summary.append(progressCounter - failedTableCounter).append(" of ").append(totalCounter) + .append(" tables synced, ").append(insertedCounter).append(" inserted, ").append(updatedCounter) + .append(" updated, ").append(skippedCounter).append(" unchanged"); + + if (failedTableCounter > 0) + summary.append(". ").append(failedTableCounter).append(" table(s) failed"); + if (failedRecordCounter > 0) + summary.append(". ").append(failedRecordCounter).append(" record(s) failed"); + if (conflictCounter > 0) + summary.append(". ").append(conflictCounter) + .append(conflictCounter == 1 ? " record is in conflict" : " records are in conflict") + .append(" and needs review before it can sync"); + + return summary.toString(); + } + /*** * Down-syncs one configured table. */ From f983c85f91248a0bf0aca37756bb53a46dce2841 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 15:31:19 +0530 Subject: [PATCH 28/81] fix: manage conflicts and add F flag for resync --- .../dataSyncActivity/DataSyncRepository.java | 8 +++++ .../DownSyncDataFromServerImpl.java | 35 +++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 97e7fcd5..9a222303 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -253,6 +253,14 @@ public Long resolveLocalIdForCentralValue(String schema, String table, String pk return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); } + public int countConflictsInLocal(String schema, String table, Object vanID) { + jdbcTemplate = getJdbcTemplate(); + String query = " SELECT COUNT(*) FROM " + schema + "." + table + + " WHERE VanID = ? AND Processed = 'F' AND SyncFailureReason = 'CONFLICT' "; + Integer count = jdbcTemplate.queryForObject(query, Integer.class, vanID); + return count == null ? 0 : count; + } + public boolean isColumnNullable(String schema, String table, String column) { jdbcTemplate = getJdbcTemplate(); List nullable = jdbcTemplate.queryForList( diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 963d4526..dfa92f42 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -103,6 +103,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { Collections.synchronizedList(new ArrayList<>()); private static DownSyncTableResult currentResult; private static String currentTable = ""; + private static int outstandingConflictCounter = 0; + private static final Map outstandingConflictTables = new LinkedHashMap<>(); public String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, Integer providerServiceMapID) throws Exception { @@ -152,6 +154,7 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer } progressCounter++; } + countOutstandingConflicts(downSyncTables, vanID); } finally { IN_PROGRESS.set(false); currentTable = ""; @@ -183,11 +186,36 @@ public Map getDownSyncStatus() { resultMap.put("recordsUpdated", updatedCounter); resultMap.put("recordsSkipped", skippedCounter); resultMap.put("conflicts", conflictCounter); - resultMap.put("conflictsPending", conflictCounter > 0); + resultMap.put("outstandingConflicts", outstandingConflictCounter); + synchronized (outstandingConflictTables) { + resultMap.put("outstandingConflictTables", new LinkedHashMap<>(outstandingConflictTables)); + } + resultMap.put("conflictsPending", conflictCounter > 0 || outstandingConflictCounter > 0); resultMap.put("summary", buildSummary()); return resultMap; } + private void countOutstandingConflicts(List downSyncTables, Integer vanID) { + outstandingConflictCounter = 0; + outstandingConflictTables.clear(); + + for (DownSyncTableDetail tableDetail : downSyncTables) { + if (!tableDetail.isTransactionalTable()) + continue; + try { + int conflicts = dataSyncRepository.countConflictsInLocal(tableDetail.getSchemaName(), + tableDetail.getTableName(), vanID); + if (conflicts > 0) { + outstandingConflictCounter += conflicts; + outstandingConflictTables.put(tableDetail.getTableName(), conflicts); + } + } catch (Exception e) { + logger.warn("Could not count outstanding conflicts for {}.{} : {}", tableDetail.getSchemaName(), + tableDetail.getTableName(), e.getMessage()); + } + } + } + private String buildSummary() { StringBuilder summary = new StringBuilder(); summary.append(progressCounter - failedTableCounter).append(" of ").append(totalCounter) @@ -200,7 +228,10 @@ private String buildSummary() { summary.append(". ").append(failedRecordCounter).append(" record(s) failed"); if (conflictCounter > 0) summary.append(". ").append(conflictCounter) - .append(conflictCounter == 1 ? " record is in conflict" : " records are in conflict") + .append(conflictCounter == 1 ? " new conflict" : " new conflicts"); + if (outstandingConflictCounter > 0) + summary.append(". ").append(outstandingConflictCounter) + .append(outstandingConflictCounter == 1 ? " record is in conflict" : " records are in conflict") .append(" and needs review before it can sync"); return summary.toString(); From 21c488c2802cd01ddf2bb5dc7f1dadca3de2daed Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 11:13:32 +0530 Subject: [PATCH 29/81] fix: pom version update --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 990e3672..142315ff 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.iemr.mmu mmu-api - 3.6.2 + 3.8.4 war MMU-API From 944f6f7dad6e7df2b99ec914748679d5cb3b56bf Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 13:00:59 +0530 Subject: [PATCH 30/81] fix: add the condition to block the download from different van --- .../DownSyncDataFromServerImpl.java | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index dfa92f42..e65d05fa 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -94,6 +94,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static int progressCounter = 0; private static int insertedCounter = 0; private static int updatedCounter = 0; + private static int transactionalFetchedCounter = 0; + private static int masterUpdatedCounter = 0; private static int conflictCounter = 0; private static int skippedCounter = 0; private static int failedTableCounter = 0; @@ -123,6 +125,8 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer progressCounter = 0; insertedCounter = 0; updatedCounter = 0; + transactionalFetchedCounter = 0; + masterUpdatedCounter = 0; conflictCounter = 0; skippedCounter = 0; failedTableCounter = 0; @@ -160,12 +164,13 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer currentTable = ""; } - logger.info("Down-sync finished : {} of {} tables succeeded, {} inserted, {} updated, {} skipped, " - + "{} conflicts, {} failed records", progressCounter - failedTableCounter, totalCounter, - insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter); + logger.info("Down-sync finished : {} of {} tables succeeded, {} records delivered for van {}, {} inserted," + + " {} updated, {} skipped, {} conflicts, {} failed records, {} master rows refreshed", + progressCounter - failedTableCounter, totalCounter, transactionalFetchedCounter, vanID, + insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter, + masterUpdatedCounter); - return conflictCounter > 0 ? "Down-sync completed with " + conflictCounter + " conflict(s) to review" - : "Down-sync completed"; + return buildSummary(); } public Map getDownSyncStatus() { @@ -191,7 +196,9 @@ public Map getDownSyncStatus() { resultMap.put("outstandingConflictTables", new LinkedHashMap<>(outstandingConflictTables)); } resultMap.put("conflictsPending", conflictCounter > 0 || outstandingConflictCounter > 0); + resultMap.put("recordsFoundForVan", transactionalFetchedCounter); resultMap.put("summary", buildSummary()); + resultMap.put("message", buildSummary()); return resultMap; } @@ -218,6 +225,17 @@ private void countOutstandingConflicts(List downSyncTables, private String buildSummary() { StringBuilder summary = new StringBuilder(); + + if (transactionalFetchedCounter == 0 && failedTableCounter == 0) { + summary.append("There is no record to down-sync for your van"); + if (outstandingConflictCounter > 0) + summary.append(". ").append(outstandingConflictCounter) + .append(outstandingConflictCounter == 1 ? " record is in conflict" + : " records are in conflict") + .append(" and needs review before it can sync"); + return summary.toString(); + } + summary.append(progressCounter - failedTableCounter).append(" of ").append(totalCounter) .append(" tables synced, ").append(insertedCounter).append(" inserted, ").append(updatedCounter) .append(" updated, ").append(skippedCounter).append(" unchanged"); @@ -299,6 +317,7 @@ private void downSyncTable(DownSyncTableDetail tableDetail, Integer vanID, Integ page++; totalFetched += dataFromCentral.size(); + transactionalFetchedCounter += dataFromCentral.size(); List acks = saveTransactionalData(tableDetail, serverColumns, vanColumns, dataFromCentral, vanID); @@ -389,7 +408,7 @@ private void upsertMasterData(DownSyncTableDetail tableDetail, List serv } int[] result = dataSyncRepository.updateLatestMasterInLocal(query, batch); - updatedCounter += result != null ? result.length : 0; + masterUpdatedCounter += result != null ? result.length : 0; logger.info("Down-synced {} master records into {}.{}", batch.size(), tableDetail.getSchemaName(), tableDetail.getTableName()); } From fd55819bc72d10a9632da9f54826fe21ad24eb93 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 14:17:13 +0530 Subject: [PATCH 31/81] fix: resolve foreign key violation --- .../dataSyncActivity/DataSyncRepository.java | 25 +++++++ .../DownSyncDataFromServerImpl.java | 72 +++++++++++++++---- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 9a222303..73eb47a8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -261,6 +261,31 @@ public int countConflictsInLocal(String schema, String table, Object vanID) { return count == null ? 0 : count; } + public List> getForeignKeysOfTable(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + String query = " SELECT k.CONSTRAINT_NAME, k.COLUMN_NAME AS CHILD_COLUMN," + + " k.REFERENCED_TABLE_SCHEMA AS PARENT_SCHEMA, k.REFERENCED_TABLE_NAME AS PARENT_TABLE," + + " k.REFERENCED_COLUMN_NAME AS PARENT_COLUMN " + + " FROM information_schema.KEY_COLUMN_USAGE k " + + " WHERE k.TABLE_SCHEMA = ? AND k.TABLE_NAME = ? AND k.REFERENCED_TABLE_NAME IS NOT NULL "; + + List> foreignKeys = jdbcTemplate.queryForList(query, schema, table); + if (foreignKeys == null) + return new ArrayList<>(); + + Map columnsPerConstraint = new LinkedHashMap<>(); + for (Map foreignKey : foreignKeys) { + columnsPerConstraint.merge(String.valueOf(foreignKey.get("CONSTRAINT_NAME")), 1, Integer::sum); + } + + List> singleColumn = new ArrayList<>(); + for (Map foreignKey : foreignKeys) { + if (columnsPerConstraint.get(String.valueOf(foreignKey.get("CONSTRAINT_NAME"))) == 1) + singleColumn.add(foreignKey); + } + return singleColumn; + } + public boolean isColumnNullable(String schema, String table, String column) { jdbcTemplate = getJdbcTemplate(); List nullable = jdbcTemplate.queryForList( diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index e65d05fa..849be15d 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -413,10 +413,7 @@ private void upsertMasterData(DownSyncTableDetail tableDetail, List serv tableDetail.getTableName()); } - /*** - * TRANSACTIONAL tables : record by record, so that a conflict on one record - * does not block the delivery of the rest. - */ + private List saveTransactionalData(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, List> dataFromCentral, Integer vanID) throws Exception { @@ -435,17 +432,13 @@ private List saveTransactionalData(DownSyncTableDetail tableD + " has neither LastModDate nor last_mod_date, so a change cannot be dated"); requireIdentifier(lastModColumn, "modification-time column", tableDetail); - Map fkMapping = parseFkColumnMapping(tableDetail); + Map fkMapping = resolveFkColumnMapping(tableDetail); Map fkCache = new LinkedHashMap<>(); List acks = new ArrayList<>(); for (Map incoming : dataFromCentral) { Map record = incoming; - // Resolved ignoring case : column names differ in case between tables - // (BenConsentID vs BenConsentId, vanSerialNo vs VanSerialNo) and a map - // lookup is case-sensitive, so taking the configured spelling literally - // yields null and every record then looks new on every run. Long centralID = toLong(record.get(resolveKeyIgnoringCase(record, pkColumn))); try { @@ -629,11 +622,62 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); } - /*** - * Parses FkColumnMapping - "childColumn:schema.parentTable", several separated - * by ';' - into an ordered map. Blank config gives an empty map, which is the - * "nothing to translate" case for almost every table. - */ + private Map resolveFkColumnMapping(DownSyncTableDetail tableDetail) throws Exception { + Map configured = parseFkColumnMapping(tableDetail); + if (!configured.isEmpty()) + return configured; + + Map mapping = new LinkedHashMap<>(); + String ownPK = tableDetail.getVanAutoIncColumnName() == null ? null + : tableDetail.getVanAutoIncColumnName().trim(); + + for (Map foreignKey : dataSyncRepository.getForeignKeysOfTable(tableDetail.getSchemaName(), + tableDetail.getTableName())) { + + String childColumn = asTrimmedString(foreignKey.get("CHILD_COLUMN")); + String parentSchema = asTrimmedString(foreignKey.get("PARENT_SCHEMA")); + String parentTable = asTrimmedString(foreignKey.get("PARENT_TABLE")); + String parentColumn = asTrimmedString(foreignKey.get("PARENT_COLUMN")); + + if (childColumn == null || parentSchema == null || parentTable == null || parentColumn == null) + continue; + if (childColumn.equalsIgnoreCase(ownPK)) + continue; + + String parentPK = downSyncedTransactionalPrimaryKey(parentSchema, parentTable); + if (parentPK == null || !parentPK.equalsIgnoreCase(parentColumn)) + continue; + + requireIdentifier(childColumn, "foreign key column", tableDetail); + mapping.put(childColumn, parentSchema + "." + parentTable); + } + + if (!mapping.isEmpty()) + logger.info("Down-sync will rewrite {} of {}.{} from the table\'s own foreign keys", mapping, + tableDetail.getSchemaName(), tableDetail.getTableName()); + + return mapping; + } + + private String downSyncedTransactionalPrimaryKey(String parentSchema, String parentTable) { + ArrayList parents = downSyncTableDetailRepo.getActiveDownSyncTableByName(parentTable); + for (DownSyncTableDetail parent : parents) { + if (!parentSchema.equalsIgnoreCase(parent.getSchemaName()) || !parent.isTransactionalTable()) + continue; + if (parent.getVanAutoIncColumnName() != null + && parent.getVanAutoIncColumnName().trim().matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + return parent.getVanAutoIncColumnName().trim(); + } + return null; + } + + private String asTrimmedString(Object value) { + if (value == null) + return null; + String trimmed = String.valueOf(value).trim(); + return trimmed.isEmpty() ? null : trimmed; + } + private Map parseFkColumnMapping(DownSyncTableDetail tableDetail) throws Exception { Map mapping = new LinkedHashMap<>(); String config = tableDetail.getFkColumnMapping(); From 4215eb835a9154475b1fba404fa767264f1415f4 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 14:35:39 +0530 Subject: [PATCH 32/81] fix: retry failed records --- .../DataSyncRepositoryCentralDownload.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 38005752..ba4692eb 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -244,9 +244,11 @@ public List> getDownSyncDataFromTable(String schema, String StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(schema) .append(".").append(table) - .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') ") + .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U', 'F') ") .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(lastModColumn) - .append(" > DownSyncDate ) ) "); + .append(" > DownSyncDate ) ) ") + .append(" AND ( DownSyncFailureReason IS NULL OR DownSyncFailureReason <> '") + .append(DownSyncRecordAck.CONFLICT).append("' ) "); List params = new ArrayList<>(); params.add(vanID); From 092499028abceebc8ead83dac74ec1ac030cc5c9 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 15:11:37 +0530 Subject: [PATCH 33/81] fix: update 'F' flag --- .../service/dataSyncActivity/DownSyncDataFromServerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 849be15d..795ff80b 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -507,7 +507,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD String reason = shorten(e.getMessage()); if (currentResult != null) currentResult.recordFailed(reason); - acks.add(DownSyncRecordAck.retryable(centralID, null, reason)); + acks.add(DownSyncRecordAck.failure(centralID, null, reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); } From 0f6837b2a0c97887d578edf6ae9a745585980980 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Fri, 4 Sep 2026 13:34:03 +0530 Subject: [PATCH 34/81] fix: sonarqube comments --- .../dataSyncActivity/DataSyncRepository.java | 59 +++++++--- .../DataSyncRepositoryCentralDownload.java | 65 +++++++---- .../validator/SqlIdentifierValidator.java | 103 ++++++++++++++++++ 3 files changed, 190 insertions(+), 37 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 73eb47a8..9fef4ece 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -41,6 +41,7 @@ import org.springframework.stereotype.Service; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo; +import com.iemr.mmu.utils.validator.SqlIdentifierValidator; /*** * @@ -193,8 +194,15 @@ public Map getLocalRecordForDownSync(String schema, String table return null; jdbcTemplate = getJdbcTemplate(); - String query = " SELECT " + autoIncColumnName + ", Processed, " + lastModColumn - + " AS LastModDate, LastDownSyncDate FROM " + schema + "." + table + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + String validLastModColumn = SqlIdentifierValidator.validatedColumnName(lastModColumn); + + String query = " SELECT " + validPkColumn + ", Processed, " + validLastModColumn + + " AS LastModDate, LastDownSyncDate FROM " + validSchema + "." + validTable + " WHERE CentralID = ? AND VanID = ? "; List> resultSet = jdbcTemplate.queryForList(query, centralID, vanID); @@ -227,7 +235,14 @@ public int updateDownSyncRecordInLocal(String query, Object[] params) { public int updateVanSerialNoInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); - String query = " UPDATE " + schema + "." + table + " SET VanSerialNo = ? WHERE " + autoIncColumnName + " = ? "; + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + + String query = " UPDATE " + validSchema + "." + validTable + " SET VanSerialNo = ? WHERE " + + validPkColumn + " = ? "; return jdbcTemplate.update(query, localID, localID); } @@ -238,24 +253,35 @@ public Long resolveLocalIdForCentralValue(String schema, String table, String pk return null; jdbcTemplate = getJdbcTemplate(); - String from = " FROM " + schema + "." + table + " WHERE "; - - Long resolved = queryForFirstLong(" SELECT " + pkColumn + from + " CentralID = ? AND VanID = ? ", centralValue, - vanID); + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumn); + String from = " FROM " + validSchema + "." + validTable + " WHERE "; + + Long resolved = queryForFirstLong(" SELECT " + validPkColumn + from + " CentralID = ? AND VanID = ? ", + centralValue, vanID); if (resolved != null) return resolved; - resolved = queryForFirstLong(" SELECT " + pkColumn + from + " VanSerialNo = ? AND VanID = ? ", centralValue, - vanID); + resolved = queryForFirstLong(" SELECT " + validPkColumn + from + " VanSerialNo = ? AND VanID = ? ", + centralValue, vanID); if (resolved != null) return resolved; - return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); + return queryForFirstLong(" SELECT " + validPkColumn + from + validPkColumn + " = ? AND VanID = ? ", + centralValue, vanID); } public int countConflictsInLocal(String schema, String table, Object vanID) { jdbcTemplate = getJdbcTemplate(); - String query = " SELECT COUNT(*) FROM " + schema + "." + table + // schema & table cannot be bound as query parameters, so both are validated + // before they are concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + + String query = " SELECT COUNT(*) FROM " + validSchema + "." + validTable + " WHERE VanID = ? AND Processed = 'F' AND SyncFailureReason = 'CONFLICT' "; Integer count = jdbcTemplate.queryForObject(query, Integer.class, vanID); return count == null ? 0 : count; @@ -302,9 +328,16 @@ private Long queryForFirstLong(String query, Object... params) { public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); - String query = " UPDATE " + schema + "." + table + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + + String query = " UPDATE " + validSchema + "." + validTable + " SET Processed = 'F', SyncFailureReason = 'CONFLICT'," - + " DownSynced = 'F', DownSyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; + + " DownSynced = 'F', DownSyncFailureReason = 'CONFLICT' WHERE " + validPkColumn + + " = ? "; return jdbcTemplate.update(query, localID); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index ba4692eb..50ea0b09 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -29,6 +29,7 @@ import javax.sql.DataSource; import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; +import com.iemr.mmu.utils.validator.SqlIdentifierValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -232,20 +233,26 @@ public List> getDownSyncDataFromTable(String schema, String if (schema == null || table == null) throw new Exception("Invalid down-sync request. Schema/table info is missing"); - String columns = (columnNames == null || columnNames.trim().isEmpty()) ? " * " : columnNames; + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String columns = (columnNames == null || columnNames.trim().isEmpty()) ? " * " + : SqlIdentifierValidator.validatedColumnList(columnNames); List> resultSetList; if (DOWN_SYNC_TRANSACTIONAL.equalsIgnoreCase(tableType)) { if (vanID == null) - throw new Exception("Invalid down-sync request. VanID is mandatory for transactional table " + table); + throw new Exception( + "Invalid down-sync request. VanID is mandatory for transactional table " + validTable); - boolean paged = pkColumn != null && pkColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$") && batchSize != null - && batchSize > 0; + String validLastModColumn = SqlIdentifierValidator.validatedColumnName(lastModColumn); + boolean paged = SqlIdentifierValidator.isValidIdentifier(pkColumn) && batchSize != null && batchSize > 0; - StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(schema) - .append(".").append(table) + StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(validSchema) + .append(".").append(validTable) .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U', 'F') ") - .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(lastModColumn) + .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(validLastModColumn) .append(" > DownSyncDate ) ) ") .append(" AND ( DownSyncFailureReason IS NULL OR DownSyncFailureReason <> '") .append(DownSyncRecordAck.CONFLICT).append("' ) "); @@ -254,22 +261,23 @@ public List> getDownSyncDataFromTable(String schema, String params.add(vanID); if (paged) { + String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumn); if (lastFetchedID != null) { - query.append(" AND ").append(pkColumn).append(" > ? "); + query.append(" AND ").append(validPkColumn).append(" > ? "); params.add(lastFetchedID); } - query.append(" ORDER BY ").append(pkColumn).append(" LIMIT ").append(batchSize.intValue()); + query.append(" ORDER BY ").append(validPkColumn).append(" LIMIT ").append(batchSize.intValue()); } - logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + logger.info("Down-sync select query for {}.{} : {}", validSchema, validTable, query); resultSetList = jdbcTemplate.queryForList(query.toString(), params.toArray()); } else { - String query = " SELECT " + columns + " FROM " + schema + "." + table; - logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + String query = " SELECT " + columns + " FROM " + validSchema + "." + validTable; + logger.info("Down-sync select query for {}.{} : {}", validSchema, validTable, query); resultSetList = jdbcTemplate.queryForList(query); } - logger.info("Down-sync record count for {}.{} : {}", schema, table, resultSetList.size()); + logger.info("Down-sync record count for {}.{} : {}", validSchema, validTable, resultSetList.size()); return resultSetList; } @@ -282,6 +290,12 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk if (records == null || records.isEmpty()) return 0; + // schema, table & primary key column cannot be bound as query parameters, so + // each of them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumnName); + List successWithSerialNo = new ArrayList<>(); List successWithoutSerialNo = new ArrayList<>(); List failed = new ArrayList<>(); @@ -306,33 +320,33 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk int updatedRows = 0; if (!successWithSerialNo.isEmpty()) { - String query = " UPDATE " + schema + "." + table + String query = " UPDATE " + validSchema + "." + validTable + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL, VanSerialNo = ? " - + " WHERE " + pkColumnName + " = ? "; + + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithSerialNo)); } if (!successWithoutSerialNo.isEmpty()) { - String query = " UPDATE " + schema + "." + table + String query = " UPDATE " + validSchema + "." + validTable + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE " - + pkColumnName + " = ? "; + + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); } if (!retryable.isEmpty()) { - String query = " UPDATE " + schema + "." + table - + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + String query = " UPDATE " + validSchema + "." + validTable + + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, retryable)); } if (!failed.isEmpty()) { - String query = " UPDATE " + schema + "." + table - + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + String query = " UPDATE " + validSchema + "." + validTable + + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, failed)); } - logger.info("Down-sync flag updated for {}.{}. Records : {}, rows : {}", schema, table, records.size(), - updatedRows); + logger.info("Down-sync flag updated for {}.{}. Records : {}, rows : {}", validSchema, validTable, + records.size(), updatedRows); return updatedRows; } @@ -341,7 +355,10 @@ public int markDownSyncedPostUpSync(String schema, String table, List return 0; jdbcTemplate = getJdbcTemplate(); - String query = " UPDATE " + schema + "." + table + // schema & table cannot be bound as query parameters, so both are validated + // before they are concatenated into the query + String query = " UPDATE " + SqlIdentifierValidator.validatedSchemaName(schema) + "." + + SqlIdentifierValidator.validatedTableName(table) + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE VanSerialNo = ? AND VanID = ? "; return countUpdatedRows(jdbcTemplate.batchUpdate(query, vanSerialNoAndVanID)); diff --git a/src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java b/src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java new file mode 100644 index 00000000..ed058a30 --- /dev/null +++ b/src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java @@ -0,0 +1,103 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +package com.iemr.mmu.utils.validator; + +import java.util.Set; +import java.util.regex.Pattern; + +/*** + * Validates the schema, table and column names that the data-sync layer has to + * concatenate into its dynamic SQL, because a schema/table/column name cannot be + * supplied as a prepared-statement parameter. Every such identifier reaching a + * query must first be passed through this class, so that nothing but a plain SQL + * identifier can ever become part of a statement. + */ +public final class SqlIdentifierValidator { + + private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); + + private static final int MAX_IDENTIFIER_LENGTH = 64; + + private static final Set VALID_SCHEMAS = Set.of("public", "db_iemr", "db_identity", "apl_db_iemr", + "apl_db_identity", "db_iemr_sync", "db_identity_sync"); + + private SqlIdentifierValidator() { + // utility class + } + + public static boolean isValidIdentifier(String identifier) { + return identifier != null && identifier.length() <= MAX_IDENTIFIER_LENGTH + && IDENTIFIER_PATTERN.matcher(identifier).matches(); + } + + /*** + * Returns the allow-listed constant matching the given schema name, so that the + * value concatenated into a query never originates from the caller. + */ + public static String validatedSchemaName(String schemaName) { + if (schemaName != null) { + for (String allowedSchema : VALID_SCHEMAS) { + if (allowedSchema.equalsIgnoreCase(schemaName)) { + return allowedSchema; + } + } + } + throw new IllegalArgumentException("Invalid schema name provided for the data-sync query"); + } + + public static String validatedIdentifier(String identifier, String identifierType) { + if (!isValidIdentifier(identifier)) { + throw new IllegalArgumentException( + "Invalid " + identifierType + " provided for the data-sync query"); + } + return identifier; + } + + public static String validatedTableName(String tableName) { + return validatedIdentifier(tableName, "table name"); + } + + public static String validatedColumnName(String columnName) { + return validatedIdentifier(columnName, "column name"); + } + + /*** + * Validates a comma separated column list, as used in the SELECT clause of the + * sync queries. Returns the trimmed, comma separated list built back from the + * validated names. + */ + public static String validatedColumnList(String columnNames) { + if (columnNames == null || columnNames.trim().isEmpty()) { + throw new IllegalArgumentException("Invalid column list provided for the data-sync query"); + } + + StringBuilder validatedColumns = new StringBuilder(); + for (String columnName : columnNames.split(",")) { + if (validatedColumns.length() > 0) { + validatedColumns.append(", "); + } + validatedColumns.append(validatedColumnName(columnName.trim())); + } + return validatedColumns.toString(); + } +} From b7005a3d29d533edb76ee68fb0aa71b2295ab00a Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Fri, 4 Sep 2026 15:50:50 +0530 Subject: [PATCH 35/81] fix: resolve sonarcube comments --- .../controller/dataSyncActivity/StartSyncActivity.java | 4 ++-- .../dataSyncLayerCentral/MMUDataSyncVanToServer.java | 8 ++++---- .../dataSyncActivity/DownSyncDataFromServerImpl.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index b517479c..a7f2a7c7 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -264,7 +264,7 @@ public String startDownSync(@RequestBody String requestOBJ, response.setError(5000, "vanID is missing, Kindly contact the administrator."); } } catch (Exception e) { - logger.error("Error in down-sync : " + e); + logger.error("Error in down-sync : {}", e.getMessage(), e); response.setError(e); } return response.toString(); @@ -277,7 +277,7 @@ public String checkDownSyncProgress() { try { response.setResponse(new Gson().toJson(downSyncDataFromServerImpl.getDownSyncStatus())); } catch (Exception e) { - logger.error("Error in down-sync progress check : " + e); + logger.error("Error in down-sync progress check : {}", e.getMessage(), e); response.setError(e); } return response.toString(); diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java index 35fb0022..589e1dcb 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java @@ -179,8 +179,8 @@ public String downSyncDataFromServer(@RequestBody DownSyncDataDigester downSyncD } } catch (Exception e) { if (downSyncDataDigester != null) - logger.error(e + " - Error in down-sync for table " + downSyncDataDigester.getSchemaName() + "." - + downSyncDataDigester.getTableName()); + logger.error("Error in down-sync for table {}.{}", downSyncDataDigester.getSchemaName(), + downSyncDataDigester.getTableName(), e); response.setError(e); } return response.toStringWithSerialization(); @@ -201,8 +201,8 @@ public String updateDownSyncFlagPostDownload(@RequestBody DownSyncDataDigester d } } catch (Exception e) { if (downSyncDataDigester != null) - logger.error(e + " - Error while updating the down-sync flag for " - + downSyncDataDigester.getSchemaName() + "." + downSyncDataDigester.getTableName()); + logger.error("Error while updating the down-sync flag for {}.{}", downSyncDataDigester.getSchemaName(), + downSyncDataDigester.getTableName(), e); response.setError(e); } return response.toStringWithSerialization(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 795ff80b..e11f1041 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -175,7 +175,7 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer public Map getDownSyncStatus() { Map resultMap = new LinkedHashMap<>(); - resultMap.put("percentage", totalCounter == 0 ? 0 : Math.floor((progressCounter * 100) / totalCounter)); + resultMap.put("percentage", totalCounter == 0 ? 0 : Math.floor((progressCounter * 100.0) / totalCounter)); resultMap.put("inProgress", IN_PROGRESS.get()); resultMap.put("currentTable", currentTable); resultMap.put("totalTables", totalCounter); From ab87e18a836260345dbc2426c18d1d1bf6373c20 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 9 Sep 2026 13:57:43 +0530 Subject: [PATCH 36/81] fix: issue in m_beneficiaryregidmapping table --- .../dataSyncActivity/DataSyncRepository.java | 44 ++++++++++++++++--- .../DownSyncDataFromServerImpl.java | 21 ++++++++- .../DataSyncRepositoryCentralDownload.java | 24 +++++++--- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 9fef4ece..75a74842 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -189,7 +189,7 @@ public List getDownSyncColumns(String schema, String table) { } public Map getLocalRecordForDownSync(String schema, String table, String autoIncColumnName, - Object centralID, Object vanID, String lastModColumn) { + Object centralID, Object vanID, String lastModColumn, boolean preserveCentralPK) { if (centralID == null) return null; @@ -201,17 +201,51 @@ public Map getLocalRecordForDownSync(String schema, String table String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); String validLastModColumn = SqlIdentifierValidator.validatedColumnName(lastModColumn); - String query = " SELECT " + validPkColumn + ", Processed, " + validLastModColumn - + " AS LastModDate, LastDownSyncDate FROM " + validSchema + "." + validTable - + " WHERE CentralID = ? AND VanID = ? "; + StringBuilder query = new StringBuilder(" SELECT ").append(validPkColumn).append(", CentralID, Processed, ") + .append(validLastModColumn).append(" AS LastModDate, LastDownSyncDate FROM ").append(validSchema) + .append(".").append(validTable); - List> resultSet = jdbcTemplate.queryForList(query, centralID, vanID); + List params = new ArrayList<>(); + + if (preserveCentralPK) { + query.append(" WHERE ( CentralID = ? OR ").append(validPkColumn).append(" = ? ) "); + params.add(centralID); + params.add(centralID); + } else { + query.append(" WHERE CentralID = ? AND VanID = ? "); + params.add(centralID); + params.add(vanID); + } + + List> resultSet = jdbcTemplate.queryForList(query.toString(), params.toArray()); if (resultSet == null || resultSet.isEmpty()) return null; return resultSet.get(0); } + /*** + * Stamps the central key on a local row that was matched by its primary key, so + * the later runs of the down-sync find it on the CentralID match itself. + */ + public int stampCentralIDInLocal(String schema, String table, String autoIncColumnName, Object localID, + Object centralID) { + if (localID == null || centralID == null) + return 0; + + jdbcTemplate = getJdbcTemplate(); + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + + String query = " UPDATE " + validSchema + "." + validTable + " SET CentralID = ? WHERE " + validPkColumn + + " = ? AND CentralID IS NULL "; + + return jdbcTemplate.update(query, centralID, localID); + } + public Long insertDownSyncRecordInLocal(final String query, final Object[] params) { jdbcTemplate = getJdbcTemplate(); KeyHolder keyHolder = new GeneratedKeyHolder(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index e11f1041..c3c1e1d0 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -72,6 +72,11 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String PROCESSED = "Processed"; private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; private static final String CENTRAL_ID = "CentralID"; + private static final String PROVISIONED = "Provisioned"; + private static final String RESERVED = "Reserved"; + private static final String RESERVED_FOR = "ReservedFor"; + private static final String RESERVED_BY_ID = "ReservedById"; + private static final String RESERVED_ON = "ReservedOn"; private static final int ACK_BATCH_SIZE = 500; @@ -450,7 +455,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD Map localRecord = dataSyncRepository.getLocalRecordForDownSync( tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, centralID, vanID, - lastModColumn); + lastModColumn, tableDetail.isPreserveCentralPK()); // a pointer at another table's primary key means a different number here // than at central, so it is rewritten before the row is written @@ -473,6 +478,11 @@ private List saveTransactionalData(DownSyncTableDetail tableD Long localID = toLong(localRecord.get(pkColumn)); + // a row matched by its primary key alone carries no central key yet + if (localRecord.get(CENTRAL_ID) == null) + dataSyncRepository.stampCentralIDInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), + pkColumn, localID, centralID); + if (!isCentralCopyNewer(record, localRecord, lastModColumn)) { skippedCounter++; if (currentResult != null) @@ -598,7 +608,8 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo for (int i = 0; i < vanColumns.size(); i++) { String vanColumn = vanColumns.get(i); - if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn) + || isVanOwnedColumn(vanColumn)) continue; if (columnsToLeaveAlone != null && containsIgnoringCase(columnsToLeaveAlone, vanColumn)) continue; @@ -816,6 +827,12 @@ private void requireIdentifier(String value, String what, DownSyncTableDetail ta + what); } + private boolean isVanOwnedColumn(String column) { + return VAN_ID.equalsIgnoreCase(column) || PROVISIONED.equalsIgnoreCase(column) + || RESERVED.equalsIgnoreCase(column) || RESERVED_FOR.equalsIgnoreCase(column) + || RESERVED_BY_ID.equalsIgnoreCase(column) || RESERVED_ON.equalsIgnoreCase(column); + } + private boolean isDownSyncManagedColumn(String column) { return VAN_SERIAL_NO.equalsIgnoreCase(column) || CENTRAL_ID.equalsIgnoreCase(column) || PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 50ea0b09..c76962b4 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -296,6 +296,11 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk String validTable = SqlIdentifierValidator.validatedTableName(table); String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumnName); + String lastModColumn = resolveLastModColumn(schema, table); + String holdLastModDate = lastModColumn == null ? "" + : ", " + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " = " + + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " "; + List successWithSerialNo = new ArrayList<>(); List successWithoutSerialNo = new ArrayList<>(); List failed = new ArrayList<>(); @@ -322,26 +327,28 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk if (!successWithSerialNo.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL, VanSerialNo = ? " - + " WHERE " + validPkColumn + " = ? "; + + holdLastModDate + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithSerialNo)); } if (!successWithoutSerialNo.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable - + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE " - + validPkColumn + " = ? "; + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + holdLastModDate + + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); } if (!retryable.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable - + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; + + " SET DownSynced = 'U', DownSyncFailureReason = ? " + holdLastModDate + " WHERE " + validPkColumn + + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, retryable)); } if (!failed.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable - + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; + + " SET DownSynced = 'F', DownSyncFailureReason = ? " + holdLastModDate + " WHERE " + validPkColumn + + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, failed)); } @@ -357,9 +364,14 @@ public int markDownSyncedPostUpSync(String schema, String table, List jdbcTemplate = getJdbcTemplate(); // schema & table cannot be bound as query parameters, so both are validated // before they are concatenated into the query + String lastModColumn = resolveLastModColumn(schema, table); + String holdLastModDate = lastModColumn == null ? "" + : ", " + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " = " + + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " "; + String query = " UPDATE " + SqlIdentifierValidator.validatedSchemaName(schema) + "." + SqlIdentifierValidator.validatedTableName(table) - + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + holdLastModDate + " WHERE VanSerialNo = ? AND VanID = ? "; return countUpdatedRows(jdbcTemplate.batchUpdate(query, vanSerialNoAndVanID)); } From 03da110af219e4dec5908cc3a1ff82eb14a31578 Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Fri, 11 Sep 2026 09:20:02 +0530 Subject: [PATCH 37/81] Adding api's for syncing diagnostic documents to s3 --- pom.xml | 6 + src/main/environment/common_ci.properties | 11 + src/main/environment/common_docker.properties | 11 + .../environment/common_example.properties | 10 + .../com/iemr/mmu/config/S3ClientConfig.java | 42 +++ .../dataSyncActivity/StartSyncActivity.java | 48 +++ .../MMUDataSyncVanToServer.java | 23 +- .../DiagnosticDocumentFetchService.java | 62 ++++ .../DiagnosticDocumentIngestService.java | 117 ++++++++ .../DiagnosticDocumentPushServiceImpl.java | 273 ++++++++++++++++++ .../DiagnosticDocumentRepository.java | 95 ++++++ .../java/com/iemr/mmu/utils/CryptoUtil.java | 42 +++ 12 files changed, 738 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/config/S3ClientConfig.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentFetchService.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentRepository.java create mode 100644 src/main/java/com/iemr/mmu/utils/CryptoUtil.java diff --git a/pom.xml b/pom.xml index 59ea5dce..27d4ffdd 100644 --- a/pom.xml +++ b/pom.xml @@ -292,6 +292,12 @@ h2 runtime + + + org.apache.commons + commons-csv + 1.11.0 + org.apache.commons diff --git a/src/main/environment/common_ci.properties b/src/main/environment/common_ci.properties index 918156fa..2e13af97 100644 --- a/src/main/environment/common_ci.properties +++ b/src/main/environment/common_ci.properties @@ -5,6 +5,12 @@ spring.datasource.username=@env.DATABASE_USERNAME@ spring.datasource.password=@env.DATABASE_PASSWORD@ spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver +## S3 storage for diagnostic documents pushed from a van +aws.s3.access-key=@env.AWS_S3_ACCESS_KEY@ +aws.s3.secret-key=@env.AWS_S3_SECRET_KEY@ +aws.s3.region=@env.AWS_S3_REGION@ +diagnostic.documents.s3.bucket=@env.DIAGNOSTIC_DOCUMENTS_S3_BUCKET@ + ## Carestream URLs, local carestreamOrderCreateURL=@env.COMMON_API@carestream/createOrder @@ -28,6 +34,11 @@ dataSyncUploadUrl=@env.MMU_CENTRAL_SERVER@dataSync/van-to-server ## Data download API, central dataSyncDownloadUrl=@env.MMU_CENTRAL_SERVER@dataSync/server-to-van +## Diagnostic document push (this server -> further central server) +diagnosticDocumentUploadUrl=@env.MMU_CENTRAL_SERVER@dataSync/diagnostic-documents +diagnosticDocument.push.batchSize=@env.DIAGNOSTIC_DOCUMENT_PUSH_BATCH_SIZE@ +diagnostic.documents.storage-root=@env.DIAGNOSTIC_DOCUMENTS_STORAGE_ROOT@ + ## TC specialist slot booking, # not required tcSpecialistSlotBook=@env.TM_API@schedule/bookSlot diff --git a/src/main/environment/common_docker.properties b/src/main/environment/common_docker.properties index 52298c5f..b11da15a 100644 --- a/src/main/environment/common_docker.properties +++ b/src/main/environment/common_docker.properties @@ -5,6 +5,12 @@ spring.datasource.username=${DATABASE_USERNAME} spring.datasource.password=${DATABASE_PASSWORD} spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver +## S3 storage for diagnostic documents pushed from a van +aws.s3.access-key=${AWS_S3_ACCESS_KEY} +aws.s3.secret-key=${AWS_S3_SECRET_KEY} +aws.s3.region=${AWS_S3_REGION} +diagnostic.documents.s3.bucket=${DIAGNOSTIC_DOCUMENTS_S3_BUCKET} + ## Carestream URLs, local carestreamOrderCreateURL=${COMMON_API}carestream/createOrder @@ -28,6 +34,11 @@ dataSyncUploadUrl=${MMU_CENTRAL_SERVER}dataSync/van-to-server ## Data download API, central dataSyncDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van +## Diagnostic document push (this server -> further central server) +diagnosticDocumentUploadUrl=${MMU_CENTRAL_SERVER}dataSync/diagnostic-documents +diagnosticDocument.push.batchSize=${DIAGNOSTIC_DOCUMENT_PUSH_BATCH_SIZE} +diagnostic.documents.storage-root=${DIAGNOSTIC_DOCUMENTS_STORAGE_ROOT} + ## TC specialist slot booking, # not required tcSpecialistSlotBook=${TM_API}schedule/bookSlot diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 7157c564..69b99ba5 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -7,6 +7,12 @@ spring.datasource.username= spring.datasource.password= spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver +## S3 storage for diagnostic documents pushed from a van (POST /dataSync/diagnostic-documents) +aws.s3.access-key= +aws.s3.secret-key= +aws.s3.region=ap-south-1 +diagnostic.documents.s3.bucket= + ## Carestream URLs, local #carestreamOrderCreateURL =http://localhost:1040/carestream/createOrder carestreamOrderCreateURL =http://localhost:8083/carestream/createOrder @@ -44,6 +50,10 @@ dataSyncUploadUrl=http://10.208.122.38:8080/mmu-api/dataSync/van-to-server #dataSyncDownloadUrl=http://localhost:82/dataSync/server-to-van dataSyncDownloadUrl=http://10.208.122.38:8080/mmu-api/dataSync/server-to-van +diagnosticDocumentUploadUrl=http://localhost:8087/mmu-api/dataSync/diagnostic-documents +diagnosticDocument.push.batchSize=3 +diagnostic.documents.storage-root= + ## TC specialist slot booking, # not required #tcSpecialistSlotBook=http://localhost:8080/schedule/bookSlot tcSpecialistSlotBook=http://10.208.122.38:8080/tm-api/schedule/bookSlot diff --git a/src/main/java/com/iemr/mmu/config/S3ClientConfig.java b/src/main/java/com/iemr/mmu/config/S3ClientConfig.java new file mode 100644 index 00000000..af02dcbb --- /dev/null +++ b/src/main/java/com/iemr/mmu/config/S3ClientConfig.java @@ -0,0 +1,42 @@ +package com.iemr.mmu.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; + +/*** + * @purpose S3 client beans used to store diagnostic documents pushed from a van. Built + * once and shared, rather than re-built per request. + */ +@Configuration +public class S3ClientConfig { + + @Value("${aws.s3.access-key}") + private String accessKey; + + @Value("${aws.s3.secret-key}") + private String secretKey; + + @Value("${aws.s3.region}") + private String region; + + private StaticCredentialsProvider credentialsProvider() { + return StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)); + } + + @Bean + public S3Client diagnosticDocumentS3Client() { + return S3Client.builder().region(Region.of(region)).credentialsProvider(credentialsProvider()).build(); + } + + @Bean + public S3Presigner diagnosticDocumentS3Presigner() { + return S3Presigner.builder().region(Region.of(region)).credentialsProvider(credentialsProvider()).build(); + } +} diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 83c5dcfe..04f23f69 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -21,6 +21,8 @@ */ package com.iemr.mmu.controller.dataSyncActivity; +import java.util.Map; + import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,12 +33,15 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import com.google.gson.Gson; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl; import com.iemr.mmu.service.dataSyncActivity.UploadDataToServerImpl; +import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentFetchService; +import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentPushServiceImpl; import com.iemr.mmu.utils.CookieUtil; import com.iemr.mmu.utils.response.OutputResponse; @@ -58,6 +63,10 @@ public class StartSyncActivity { private DownloadDataFromServerImpl downloadDataFromServerImpl; @Autowired private DownloadDataFromServerTransactionalImpl downloadDataFromServerTransactionalImpl; + @Autowired + private DiagnosticDocumentPushServiceImpl diagnosticDocumentPushServiceImpl; + @Autowired + private DiagnosticDocumentFetchService diagnosticDocumentFetchService; private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; @@ -88,6 +97,45 @@ public String dataSyncToServer(@RequestBody String requestOBJ, return response.toStringWithSerialization(); } + @Operation(summary = "Push pending diagnostic documents (docsProcessed='N') to the further central server, in batches") + @PostMapping(value = { "/diagnostic-documents-to-server" }) + public String diagnosticDocumentsToServer(@RequestHeader(value = "Authorization") String authorization, + @RequestHeader(value = "ServerAuthorization") String serverAuthorization, + @RequestParam(required = false) Long villageId) { + OutputResponse response = new OutputResponse(); + try { + String s = diagnosticDocumentPushServiceImpl.pushPendingDocuments(serverAuthorization, villageId); + if (s != null) + response.setResponse(s); + else + response.setError(5000, "Error in diagnostic document push"); + } catch (Exception e) { + logger.error("Error in diagnostic document push : " + e); + response.setError(e); + } + return response.toString(); + } + + @Operation(summary = "Fetch a short-lived download URL for the latest successfully-pushed diagnostic document matching a beneficiary+documentType, from this van's own local record") + @GetMapping(value = { "/diagnostic-documents/download" }) + public String diagnosticDocumentDownloadUrl(@RequestParam Long beneficiaryId, @RequestParam String documentType, + @RequestHeader(value = "Authorization") String authorization) { + OutputResponse response = new OutputResponse(); + try { + Map download = diagnosticDocumentFetchService.getLatestDocumentDownload(beneficiaryId, + documentType); + if (download != null) + response.setResponse(new Gson().toJson(download)); + else + response.setError(5000, "No pushed document found for beneficiaryId=" + beneficiaryId + + ", documentType=" + documentType); + } catch (Exception e) { + logger.error("Error fetching diagnostic document download URL : " + e); + response.setError(e); + } + return response.toString(); + } + @Operation(summary = "Get data sync group details") @GetMapping(value = { "/getSyncGroupDetails" }) public String getSyncGroupDetails() { diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java index ea943d0e..4fd074c2 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java @@ -24,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -34,10 +33,10 @@ import com.iemr.mmu.data.syncActivity_syncLayer.SyncDownloadMaster; import com.iemr.mmu.data.syncActivity_syncLayer.SyncUploadDataDigester; +import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentIngestService; import com.iemr.mmu.service.dataSyncLayerCentral.FetchDownloadDataImpl; import com.iemr.mmu.service.dataSyncLayerCentral.GetDataFromVanAndSyncToDBImpl; import com.iemr.mmu.service.dataSyncLayerCentral.GetMasterDataFromCentralForVanImpl; -import com.iemr.mmu.utils.CookieUtil; import com.iemr.mmu.utils.response.OutputResponse; import io.swagger.v3.oas.annotations.Operation; @@ -57,6 +56,8 @@ public class MMUDataSyncVanToServer { private GetMasterDataFromCentralForVanImpl getMasterDataFromCentralForVanImpl; @Autowired private FetchDownloadDataImpl fetchDownloadDataImpl; + @Autowired + private DiagnosticDocumentIngestService diagnosticDocumentIngestService; @Operation(summary = "Sync data from van-to-server") @PostMapping(value = { "/van-to-server" }, consumes = "application/json", produces = "application/json") @@ -77,6 +78,24 @@ public String dataSyncToServer(@RequestBody String requestOBJ, return response.toString(); } + @Operation(summary = "Receive diagnostic documents pushed from a van and store each in S3 (no database write here)") + @PostMapping(value = { "/diagnostic-documents" }, consumes = "application/json", produces = "application/json") + public String diagnosticDocumentsFromVan(@RequestBody String requestOBJ, + @RequestHeader(value = "Authorization") String Authorization) { + OutputResponse response = new OutputResponse(); + try { + String s = diagnosticDocumentIngestService.ingestDocuments(requestOBJ); + if (s != null) + response.setResponse(s); + else + response.setError(5000, "diagnostic document ingest failed"); + } catch (Exception e) { + response.setError(e); + logger.error("Diagnostic document ingest Exception" + e); + } + return response.toString(); + } + @Operation(summary = "Download data from server-to-van") @PostMapping(value = { "/server-to-van" }, consumes = "application/json", produces = "application/json") public String dataDownloadFromServer(@RequestBody SyncDownloadMaster syncDownloadMaster, diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentFetchService.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentFetchService.java new file mode 100644 index 00000000..a03526f9 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentFetchService.java @@ -0,0 +1,62 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.time.Duration; +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; + +/*** + * @purpose Hands back a short-lived presigned URL for the latest successfully-pushed + * diagnostic document matching a beneficiary+documentType, generated on demand from + * the object's S3 key (s3_path) - the bucket is private, so no permanent URL is + * ever persisted or handed out. + */ +@Service +public class DiagnosticDocumentFetchService { + + private static final Duration URL_VALIDITY = Duration.ofMinutes(15); + + @Value("${diagnostic.documents.s3.bucket}") + private String bucket; + + @Autowired + private DiagnosticDocumentRepository diagnosticDocumentRepository; + + @Autowired + private S3Presigner s3Presigner; + + /*** + * @return null if no successfully-pushed document matches, otherwise the download details + * (documentType, orderType, externalOrderId, contentType, lastModDate, downloadUrl, + * urlExpiresInSeconds) + */ + public Map getLatestDocumentDownload(Long beneficiaryId, String documentType) { + Map row = diagnosticDocumentRepository.findLatestDocument(beneficiaryId, documentType); + if (row == null) { + return null; + } + + String s3Key = (String) row.get("s3_path"); + GetObjectRequest getObjectRequest = GetObjectRequest.builder().bucket(bucket).key(s3Key).build(); + GetObjectPresignRequest presignRequest = GetObjectPresignRequest.builder().signatureDuration(URL_VALIDITY) + .getObjectRequest(getObjectRequest).build(); + String downloadUrl = s3Presigner.presignGetObject(presignRequest).url().toString(); + + Map result = new java.util.HashMap<>(); + result.put("externalOrderId", row.get("external_order_id")); + result.put("orderType", row.get("order_type")); + result.put("documentType", row.get("document_type")); + result.put("contentType", row.get("content_type")); + result.put("originalFileName", row.get("original_file_name")); + result.put("lastModDate", String.valueOf(row.get("last_mod_date"))); + result.put("downloadUrl", downloadUrl); + result.put("urlExpiresInSeconds", URL_VALIDITY.getSeconds()); + return result; + } +} \ No newline at end of file diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java new file mode 100644 index 00000000..b8e596b5 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java @@ -0,0 +1,117 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.lang.reflect.Type; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.ServerSideEncryption; + +/*** + * @purpose Receives a batch of decrypted diagnostic documents pushed from a van and stores + * each in S3 - purely a storage relay, no database writes here. The pushing van + * persists its own record locally (DiagnosticDocumentPushServiceImpl.markPushedToCentral, + * keyed off the s3Path this returns in each ack) - the central server's own database is + * left untouched. + */ +@Service +public class DiagnosticDocumentIngestService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private static final Gson GSON = new Gson(); + + @Value("${diagnostic.documents.s3.bucket}") + private String bucket; + + @Autowired + private S3Client s3Client; + + public String ingestDocuments(String requestOBJ) throws Exception { + Type listType = new TypeToken>>() { + }.getType(); + List> items = GSON.fromJson(requestOBJ, listType); + if (items == null || items.isEmpty()) { + return GSON.toJson(new ArrayList<>()); + } + + List> acks = new ArrayList<>(); + for (Map item : items) { + acks.add(ingestOne(item)); + } + return GSON.toJson(acks); + } + + private Map ingestOne(Map item) { + Long diagnosticOrderId = asLong(item.get("diagnosticOrderId")); + String externalOrderId = (String) item.get("externalOrderId"); + String documentType = (String) item.get("documentType"); + + Map ack = new HashMap<>(); + ack.put("diagnosticOrderId", diagnosticOrderId); + ack.put("externalOrderId", externalOrderId); + ack.put("documentType", documentType); + + try { + byte[] plaintext = Base64.getDecoder().decode((String) item.get("fileContentBase64")); + + String sha256Hash = (String) item.get("sha256Hash"); + if (sha256Hash != null && !sha256Hash.equalsIgnoreCase(sha256Hex(plaintext))) { + ack.put("status", "FAILED"); + ack.put("error", "sha256 mismatch on receipt"); + return ack; + } + + Long beneficiaryId = asLong(item.get("beneficiaryId")); + Long villageId = asLong(item.get("villageId")); + String orderType = (String) item.get("orderType"); + String storedFileName = (String) item.get("storedFileName"); + String s3Key = villageId + "/" + beneficiaryId + "/" + orderType + "/" + documentType + "/" + + storedFileName; + String contentType = (String) item.get("contentType"); + + s3Client.putObject( + PutObjectRequest.builder().bucket(bucket).key(s3Key) + .contentType(contentType != null ? contentType : "application/octet-stream") + .serverSideEncryption(ServerSideEncryption.AES256).build(), + RequestBody.fromBytes(plaintext)); + + ack.put("status", "SUCCESS"); + ack.put("s3Path", s3Key); + } catch (Exception e) { + logger.error("Error ingesting diagnostic document: diagnosticOrderId=" + diagnosticOrderId + + ", documentType=" + documentType, e); + ack.put("status", "FAILED"); + ack.put("error", e.getMessage()); + } + return ack; + } + + private static Long asLong(Object value) { + return value == null ? null : ((Number) value).longValue(); + } + + private static String sha256Hex(byte[] data) throws Exception { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(data); + StringBuilder hex = new StringBuilder(digest.length * 2); + for (byte b : digest) { + hex.append(String.format(Locale.ROOT, "%02x", b)); + } + return hex.toString(); + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java new file mode 100644 index 00000000..adb7714c --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java @@ -0,0 +1,273 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; +import com.iemr.mmu.utils.CryptoUtil; +import com.iemr.mmu.utils.RestTemplateUtil; + +import java.lang.reflect.Type; + +/*** + * @purpose Reads MMU-API's own locally-pending diagnostic documents (docsProcessed='N', + * shared db_iemr.tb_diagnostic_document table), decrypts the file off the shared + * filesystem, and pushes each batch to the further central server's + * /dataSync/diagnostic-documents endpoint - the same relay shape as + * UploadDataToServerImpl's push to dataSyncUploadUrl, just for this table instead + * of the generic sync-group config. MMU-API's own /dataSync/diagnostic-documents + * ingest endpoint (DiagnosticDocumentCentralIngestService) is untouched by this - + * this service is a separate outbound relay, not a caller of it. + */ +@Service +public class DiagnosticDocumentPushServiceImpl { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private static final Map CONTENT_TYPE_EXTENSIONS = new HashMap<>(); + static { + CONTENT_TYPE_EXTENSIONS.put("application/pdf", "pdf"); + CONTENT_TYPE_EXTENSIONS.put("image/jpeg", "jpg"); + CONTENT_TYPE_EXTENSIONS.put("image/png", "png"); + } + + @Value("${diagnostic.documents.storage-root}") + private String storageRoot; + + @Value("${diagnosticDocumentUploadUrl}") + private String diagnosticDocumentUploadUrl; + + @Value("${diagnosticDocument.push.batchSize:3}") + private int batchSize; + + @Autowired + private DiagnosticDocumentRepository diagnosticDocumentRepository; + + @Autowired + private CryptoUtil cryptoUtil; + + public String pushPendingDocuments(String Authorization, Long villageId) throws Exception { + List> pendingRows = diagnosticDocumentRepository.findPendingDocuments(); + boolean anyRowsFound = !pendingRows.isEmpty(); + int totalAttempted = 0; + int totalSucceeded = 0; + + for (int offset = 0; offset < pendingRows.size(); offset += batchSize) { + List> rows = pendingRows.subList(offset, Math.min(offset + batchSize, pendingRows.size())); + + List> payloadItems = new ArrayList<>(); + Map> rowsByAckKey = new HashMap<>(); + for (Map row : rows) { + Long rowId = asLong(row.get("id")); + String base64Plaintext; + try { + String storedPath = (String) row.get("stored_path"); + Path filePath = Paths.get(storageRoot, storedPath); + String encryptedPayload = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8); + base64Plaintext = cryptoUtil.decrypt(encryptedPayload); + } catch (Exception e) { + // File missing/unreadable off the shared filesystem - mark just this row + // failed and move on, rather than letting the exception abort the whole run + // (which would leave every other pending row, in this batch and beyond, + // completely untouched). + logger.warn("Skipping diagnostic document push, could not read file off disk: id={}, error={}", + rowId, e.getMessage()); + diagnosticDocumentRepository.markPushFailed(rowId, + "Could not read file off shared filesystem: " + e.getMessage()); + continue; + } + if (base64Plaintext == null) { + logger.warn("Skipping diagnostic document push, decrypt failed: id={}", rowId); + diagnosticDocumentRepository.markPushFailed(rowId, "Decrypt failed"); + continue; + } + + Long diagnosticOrderId = asLong(row.get("diagnostic_order_id")); + String externalOrderId = (String) row.get("external_order_id"); + String documentType = (String) row.get("document_type"); + String contentType = (String) row.get("content_type"); + + Map item = new HashMap<>(); + item.put("diagnosticOrderId", diagnosticOrderId); + item.put("externalOrderId", externalOrderId); + item.put("beneficiaryId", row.get("beneficiary_id")); + item.put("orderType", row.get("order_type")); + item.put("documentType", documentType); + item.put("storedFileName", row.get("stored_file_name")); + item.put("sha256Hash", row.get("sha256_hash")); + item.put("contentType", contentType); + item.put("fileExtension", extensionFor(contentType)); + item.put("originalFileName", row.get("original_file_name")); + item.put("vanID", row.get("vanID")); + item.put("parkingPlaceID", row.get("parkingPlaceID")); + item.put("vanSerialNo", row.get("vanSerialNo")); + item.put("villageId", villageId); + item.put("fileContentBase64", base64Plaintext); + payloadItems.add(item); + rowsByAckKey.put(ackKey(externalOrderId, documentType), row); + } + + if (payloadItems.isEmpty()) { + // Every row in this batch already got marked failed above (missing file or + // failed decrypt) - nothing left to send. + continue; + } + + // Counted here, not after the central round-trip below - these documents WERE + // successfully decrypted and queued for sending regardless of whether the central + // server subsequently accepts, rejects, or fails to respond to them. Otherwise a + // batch that decrypted fine but got rejected by the central server (e.g. an expired + // session) would leave totalAttempted at 0, and the caller would see the misleading + // "No documents could be decrypted for push" instead of the real per-row reason + // (already recorded in docSyncFailureReason). + totalAttempted += payloadItems.size(); + + String requestOBJ = new Gson().toJson(payloadItems); + List> acks; + try { + HttpEntity request = RestTemplateUtil.createRequestEntity(requestOBJ, Authorization, "datasync"); + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity response = restTemplate.exchange(diagnosticDocumentUploadUrl, HttpMethod.POST, + request, String.class); + + if (response == null || !response.hasBody()) { + logger.warn("No response body from central server for diagnostic document push, marking batch failed"); + markBatchFailed(rowsByAckKey, "No response body from central server"); + continue; + } + + // Central server wraps every response in the shared OutputResponse envelope + // ({"data": [...], "statusCode":200, ...}) - the ack array lives under "data". + JsonElement parsedBody = JsonParser.parseString(response.getBody()); + if (!parsedBody.isJsonObject()) { + logger.warn("Unexpected response shape from central server for diagnostic document push, marking batch failed"); + markBatchFailed(rowsByAckKey, "Unexpected response shape from central server"); + continue; + } + JsonObject envelope = parsedBody.getAsJsonObject(); + if (!envelope.has("statusCode") || envelope.get("statusCode").getAsInt() != 200 + || !envelope.has("data")) { + logger.warn("Central server reported failure for diagnostic document push batch, marking batch failed: {}", + response.getBody()); + markBatchFailed(rowsByAckKey, "Central server reported failure: " + response.getBody()); + continue; + } + + Type ackListType = new TypeToken>>() { + }.getType(); + acks = new Gson().fromJson(envelope.get("data"), ackListType); + if (acks == null) { + markBatchFailed(rowsByAckKey, "Central server returned no acknowledgements"); + continue; + } + } catch (Exception e) { + logger.error("Error calling central server for diagnostic document push, marking batch failed", e); + markBatchFailed(rowsByAckKey, "Error calling central server: " + e.getMessage()); + continue; + } + + int batchSuccessCount = 0; + Map> unmatchedRowsByAckKey = new HashMap<>(rowsByAckKey); + for (Map ack : acks) { + String key = ackKey((String) ack.get("externalOrderId"), (String) ack.get("documentType")); + Map row = rowsByAckKey.get(key); + if (row == null) { + continue; + } + unmatchedRowsByAckKey.remove(key); + if ("SUCCESS".equalsIgnoreCase((String) ack.get("status"))) { + diagnosticDocumentRepository.markPushedToCentral(asLong(row.get("id")), + (String) ack.get("s3Path")); + batchSuccessCount++; + } else { + String reason = (String) ack.get("error"); + logger.warn("Central server rejected diagnostic document push: externalOrderId={}, documentType={}, error={}", + ack.get("externalOrderId"), ack.get("documentType"), reason); + diagnosticDocumentRepository.markPushFailed(asLong(row.get("id")), + reason != null ? reason : "Central server rejected the document"); + } + } + if (!unmatchedRowsByAckKey.isEmpty()) { + // Central sent back fewer acks than documents we sent - whatever wasn't + // accounted for must not be silently left at its previous status forever. + logger.warn( + "Diagnostic document push: {} row(s) in this batch got no matching ack back, marking failed", + unmatchedRowsByAckKey.size()); + markBatchFailed(unmatchedRowsByAckKey, "No acknowledgement received from central server"); + } + + totalSucceeded += batchSuccessCount; + logger.info("Diagnostic document push batch complete: attempted={}, succeeded={}", payloadItems.size(), + batchSuccessCount); + } + + if (!anyRowsFound) { + return "No pending diagnostic documents to sync"; + } + if (totalAttempted == 0) { + return "No documents could be decrypted for push"; + } + if (totalSucceeded == 0) { + // Documents WERE decrypted and sent, but none were accepted (e.g. the central + // server rejected every batch) - the specific reason for each row is recorded in + // its own docSyncFailureReason, this is just the overall-outcome summary. + return "Documents were sent but none were accepted by the central server"; + } + + logger.info("Diagnostic document push complete overall: attempted={}, succeeded={}", totalAttempted, + totalSucceeded); + return "Data successfully synced"; + } + + private static String ackKey(String externalOrderId, String documentType) { + return externalOrderId + "|" + documentType; + } + + /*** + * @purpose Called when the whole batch call to the central server fails (no/garbled + * response, non-200, or a thrown exception) - marks every row that was in this + * batch as failed, matching how /van-to-server marks a whole batch failed on a + * connection error, rather than leaving them stuck at docsProcessed='N' forever. + */ + private void markBatchFailed(Map> rowsByAckKey, String reason) { + for (Map row : rowsByAckKey.values()) { + diagnosticDocumentRepository.markPushFailed(asLong(row.get("id")), reason); + } + } + + private static Long asLong(Object value) { + return value == null ? null : ((Number) value).longValue(); + } + + private static String extensionFor(String contentType) { + if (contentType == null) { + return "bin"; + } + String extension = CONTENT_TYPE_EXTENSIONS.get(contentType.toLowerCase()); + if (extension != null) { + return extension; + } + int slashIndex = contentType.indexOf('/'); + return slashIndex >= 0 && slashIndex < contentType.length() - 1 ? contentType.substring(slashIndex + 1) : "bin"; + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentRepository.java new file mode 100644 index 00000000..53c30c5f --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentRepository.java @@ -0,0 +1,95 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Service; + +/*** + * @purpose Local (van) side persistence for the diagnostic-document push pipeline - the central + * server is a pure S3 storage relay (DiagnosticDocumentCentralIngestService) and writes + * nothing to its own database, so this repository only ever operates on the pushing + * van's own local db_iemr.tb_diagnostic_document rows. + */ +@Service +public class DiagnosticDocumentRepository { + + @Autowired + private DataSource dataSource; + + private JdbcTemplate jdbcTemplate; + + private JdbcTemplate getJdbcTemplate() { + if (this.jdbcTemplate == null) { + this.jdbcTemplate = new JdbcTemplate(dataSource); + } + return this.jdbcTemplate; + } + + /*** + * @purpose Latest successfully-pushed document for a beneficiary+documentType, used to hand + * back a fresh presigned URL on demand instead of persisting a permanent (and, for a + * private bucket, non-functional) URL. s3_path holds the S3 key for a + * docsProcessed='P' row (markPushedToCentral() below writes it there). + */ + public Map findLatestDocument(Long beneficiaryId, String documentType) { + String query = "SELECT id, external_order_id, order_type, document_type, s3_path, content_type, " + + "original_file_name, last_mod_date FROM db_iemr.tb_diagnostic_document " + + "WHERE beneficiary_id = ? AND document_type = ? AND docsProcessed = 'P' " + + "ORDER BY last_mod_date DESC, id DESC LIMIT 1"; + List> rows = getJdbcTemplate().queryForList(query, beneficiaryId, documentType); + return rows.isEmpty() ? null : rows.get(0); + } + + /*** + * @purpose Snapshot of every locally-pending row's metadata (docsProcessed='N' or 'F') at + * the start of a push run, taken in one query so a row this same run marks 'F' + * partway through (e.g. an auth failure) is never re-picked-up later in the same + * run - it only becomes eligible again on the next trigger, alongside whatever's + * newly 'N' by then. Only metadata is loaded here (id, hashes, filenames, etc.) - + * each row's actual file content is still read off disk and decrypted one batch at + * a time by the caller, so the whole backlog's decrypted content is never held in + * memory at once. + */ + public List> findPendingDocuments() { + String query = "SELECT id, diagnostic_order_id, external_order_id, beneficiary_id, order_type, document_type, " + + "stored_file_name, stored_path, sha256_hash, content_type, original_file_name, " + + "vanID, parkingPlaceID, vanSerialNo FROM db_iemr.tb_diagnostic_document " + + "WHERE docsProcessed = 'N' OR docsProcessed = 'F' ORDER BY id ASC"; + return getJdbcTemplate().queryForList(query); + } + + /*** + * @purpose Marks a document as successfully pushed on the LOCAL (van) row - the central + * server itself writes nothing to its own database. s3Path is the S3 key the + * central server's ack reported back, persisted here so the van's own local DB + * knows where the document ended up, not just that it did. + */ + public void markPushedToCentral(Long id, String s3Path) { + String update = "UPDATE db_iemr.tb_diagnostic_document SET processed = 'N', docsProcessed = 'P', " + + "s3_path = ?, docSyncedDate = NOW(), docSyncFailureReason = NULL, last_mod_date = NOW() WHERE id = ?"; + getJdbcTemplate().update(update, s3Path, id); + } + + /*** + * @purpose Marks a document push to the further central server as failed - docsProcessed='F', + * matching the P/F status convention used by the generic sync pipeline's + * DataSyncRepository.updateProcessedFlagInVan for /van-to-server. Persists why in + * docSyncFailureReason (mirroring the generic pipeline's own SyncFailureReason, kept + * separate since this pipeline's docsProcessed/docSyncedDate are their own dedicated + * columns) so a failure isn't only visible in the application log. Also clears + * s3_path - otherwise a row that succeeded once, then got re-attempted and failed, + * would keep showing a stale S3 key while docsProcessed says 'F'. docSyncedDate is + * left untouched - it records the last time this row was actually confirmed synced, + * if ever. + */ + public void markPushFailed(Long id, String reason) { + String update = "UPDATE db_iemr.tb_diagnostic_document SET processed = 'N', docsProcessed = 'F', " + + "s3_path = NULL, docSyncFailureReason = ?, last_mod_date = NOW() WHERE id = ?"; + getJdbcTemplate().update(update, reason, id); + } +} diff --git a/src/main/java/com/iemr/mmu/utils/CryptoUtil.java b/src/main/java/com/iemr/mmu/utils/CryptoUtil.java new file mode 100644 index 00000000..c964fdf2 --- /dev/null +++ b/src/main/java/com/iemr/mmu/utils/CryptoUtil.java @@ -0,0 +1,42 @@ +package com.iemr.mmu.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +/** + * Decrypts diagnostic-document files written by FLW-API's own CryptoUtil + * (same AES/ECB/PKCS5Padding scheme and key) so they can be read directly + * off the shared filesystem before pushing to the further central server. + */ +@Service +public class CryptoUtil { + + private static final Logger logger = LoggerFactory.getLogger(CryptoUtil.class); + private static final String ALGORITHM = "AES"; + private static final String SECRET_KEY = "dev-envro-secret"; + + public String decrypt(String encryptedValue) { + try { + SecretKey secretKey = new SecretKeySpec(SECRET_KEY.getBytes(StandardCharsets.UTF_8), ALGORITHM); + Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); + cipher.init(Cipher.DECRYPT_MODE, secretKey); + byte[] decryptedBytes = cipher.doFinal(Base64.getDecoder().decode(encryptedValue)); + return removePadding(new String(decryptedBytes, StandardCharsets.UTF_8)); + } catch (Exception e) { + logger.error("Exception while decrypting diagnostic document", e); + return null; + } + } + + private String removePadding(String value) { + int paddingLength = value.charAt(value.length() - 1); + return value.substring(0, value.length() - paddingLength); + } +} From 0782c44a0d3f23728b78bc85f848864bb6c6d024 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 20 Aug 2026 16:00:41 +0530 Subject: [PATCH 38/81] fix: add the endpoints and schema for downsync from central to local --- src/main/environment/common_ci.properties | 9 + src/main/environment/common_docker.properties | 203 +++++++++--------- .../environment/common_example.properties | 9 + .../DownSyncTableDetail.java | 185 ++++++++++++++++ .../DownSyncTableDetailRepo.java | 41 ++++ 5 files changed, 350 insertions(+), 97 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java create mode 100644 src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java diff --git a/src/main/environment/common_ci.properties b/src/main/environment/common_ci.properties index 2e13af97..34875d64 100644 --- a/src/main/environment/common_ci.properties +++ b/src/main/environment/common_ci.properties @@ -62,6 +62,15 @@ dataSyncTransactionDownloadUrl=@env.MMU_CENTRAL_SERVER@dataSync/server-to-van-tr ## inventory transaction processed flag update URL, central, mmu dataSyncProcessedFlagUpdate=@env.MMU_CENTRAL_SERVER@dataSync/updateProcessedFlagPostDownload +## down-sync (central -> local) data API, central, mmu +downSyncDataUrl=@env.MMU_CENTRAL_SERVER@dataSync/server-to-van-downsync + +## down-sync (central -> local) flag update API, central, mmu +downSyncFlagUpdateUrl=@env.MMU_CENTRAL_SERVER@dataSync/updateDownSyncFlagPostDownload + +## mark an up-synced record as already delivered to the van (central only) +downSync.markDeliveredOnUpSync=true + fileBasePath =@env.MMU_FILE_BASE_PATH@ diff --git a/src/main/environment/common_docker.properties b/src/main/environment/common_docker.properties index b11da15a..be5f3608 100644 --- a/src/main/environment/common_docker.properties +++ b/src/main/environment/common_docker.properties @@ -1,98 +1,107 @@ -# local env -# DB Connections -spring.datasource.url=${DATABASE_URL} -spring.datasource.username=${DATABASE_USERNAME} -spring.datasource.password=${DATABASE_PASSWORD} -spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver - -## S3 storage for diagnostic documents pushed from a van -aws.s3.access-key=${AWS_S3_ACCESS_KEY} -aws.s3.secret-key=${AWS_S3_SECRET_KEY} -aws.s3.region=${AWS_S3_REGION} -diagnostic.documents.s3.bucket=${DIAGNOSTIC_DOCUMENTS_S3_BUCKET} - -## Carestream URLs, local -carestreamOrderCreateURL=${COMMON_API}carestream/createOrder - -## Identity - Common URLs -# Registration -registrationUrl =${COMMON_API}beneficiary/createBeneficiary - -registrarQuickSearchByIdUrl =${COMMON_API}beneficiary/searchUserByID - -registrarQuickSearchByPhoneNoUrl =${COMMON_API}beneficiary/searchUserByPhone - -getBenImageFromIdentity =${IDENTITY_API_BASE_URL}id/benImageByBenRegID - -beneficiaryEditUrl =${COMMON_API}beneficiary/updateBenefciaryDetails - -registrarAdvanceSearchUrl =${COMMON_API}beneficiary/searchBeneficiary - -## Data Sync API, central -dataSyncUploadUrl=${MMU_CENTRAL_SERVER}dataSync/van-to-server - -## Data download API, central -dataSyncDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van - -## Diagnostic document push (this server -> further central server) -diagnosticDocumentUploadUrl=${MMU_CENTRAL_SERVER}dataSync/diagnostic-documents -diagnosticDocument.push.batchSize=${DIAGNOSTIC_DOCUMENT_PUSH_BATCH_SIZE} -diagnostic.documents.storage-root=${DIAGNOSTIC_DOCUMENTS_STORAGE_ROOT} - -## TC specialist slot booking, # not required -tcSpecialistSlotBook=${TM_API}schedule/bookSlot - -## TC specialist slot cancelling, # not required -tcSpecialistSlotCancel=${TM_API}schedule/cancelBookedSlot - -## benGenUrl, common, central -benGenUrlCentral=${COMMON_API_OPEN_SYNC}beneficiary/generateBeneficiaryIDs - -## benImportUrl, identity, local -benImportUrlLocal=${IDENTITY_API_BASE_URL}id/saveGeneratedBenIDToLocalServer -mmucentralserver = ${MMU_CENTRAL_SERVER}common/get/Case-sheet/centralServerTMCaseSheet - -tmCentralServer = ${TM_CENTRAL_SERVER}common/get/Case-sheet/printData - -specialistSign=${COMMON_API}signature1/getSignClass - -## inventory transaction download URL, central, mmu -dataSyncTransactionDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van-transactional - -## inventory transaction processed flag update URL, central, mmu -dataSyncProcessedFlagUpdate=${MMU_CENTRAL_SERVER}dataSync/updateProcessedFlagPostDownload - - -fileBasePath =${MMU_FILE_BASE_PATH} - -## data sync batch size -BATCH_SIZE=30 -snomedCTPageSize=10 -nurseWL=7 -docWL=7 -pharmaWL=7 -labWL=7 -radioWL=7 -oncoWL=7 - -## For File Sync -serverIP=${FILE_SYNC_SERVER_IP} -serverDomain=${FILE_SYNC_SERVER_DOMAIN} -serverUserName=${FILE_SYNC_SERVER_USERNAME} -serverPassword=${FILE_SYNC_SERVER_PASSWORD} -localFolderToSync=${LOCAL_FOLDER_TO_SYNC} -serverFolder=MMUSync -getServerCredentialURL=${MMU_API}fileSyncController/getServerCredential -tmReferCheckValue=TM -### Redis IP -spring.redis.host=${REDIS_HOST} - -jwt.secret=${JWT_SECRET_KEY} - - -#ELK logging file name -logging.path=logs -logging.file.name=${MMU_API_LOGGING_FILE_NAME} - -springdoc.api-docs.enabled=${SWAGGER_DOC_ENABLED} +# local env +# DB Connections +spring.datasource.url=${DATABASE_URL} +spring.datasource.username=${DATABASE_USERNAME} +spring.datasource.password=${DATABASE_PASSWORD} +spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver + +## S3 storage for diagnostic documents pushed from a van +aws.s3.access-key=${AWS_S3_ACCESS_KEY} +aws.s3.secret-key=${AWS_S3_SECRET_KEY} +aws.s3.region=${AWS_S3_REGION} +diagnostic.documents.s3.bucket=${DIAGNOSTIC_DOCUMENTS_S3_BUCKET} + +## Carestream URLs, local +carestreamOrderCreateURL=${COMMON_API}carestream/createOrder + +## Identity - Common URLs +# Registration +registrationUrl =${COMMON_API}beneficiary/createBeneficiary + +registrarQuickSearchByIdUrl =${COMMON_API}beneficiary/searchUserByID + +registrarQuickSearchByPhoneNoUrl =${COMMON_API}beneficiary/searchUserByPhone + +getBenImageFromIdentity =${IDENTITY_API_BASE_URL}id/benImageByBenRegID + +beneficiaryEditUrl =${COMMON_API}beneficiary/updateBenefciaryDetails + +registrarAdvanceSearchUrl =${COMMON_API}beneficiary/searchBeneficiary + +## Data Sync API, central +dataSyncUploadUrl=${MMU_CENTRAL_SERVER}dataSync/van-to-server + +## Data download API, central +dataSyncDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van + +## Diagnostic document push (this server -> further central server) +diagnosticDocumentUploadUrl=${MMU_CENTRAL_SERVER}dataSync/diagnostic-documents +diagnosticDocument.push.batchSize=${DIAGNOSTIC_DOCUMENT_PUSH_BATCH_SIZE} +diagnostic.documents.storage-root=${DIAGNOSTIC_DOCUMENTS_STORAGE_ROOT} + +## TC specialist slot booking, # not required +tcSpecialistSlotBook=${TM_API}schedule/bookSlot + +## TC specialist slot cancelling, # not required +tcSpecialistSlotCancel=${TM_API}schedule/cancelBookedSlot + +## benGenUrl, common, central +benGenUrlCentral=${COMMON_API_OPEN_SYNC}beneficiary/generateBeneficiaryIDs + +## benImportUrl, identity, local +benImportUrlLocal=${IDENTITY_API_BASE_URL}id/saveGeneratedBenIDToLocalServer +mmucentralserver = ${MMU_CENTRAL_SERVER}common/get/Case-sheet/centralServerTMCaseSheet + +tmCentralServer = ${TM_CENTRAL_SERVER}common/get/Case-sheet/printData + +specialistSign=${COMMON_API}signature1/getSignClass + +## inventory transaction download URL, central, mmu +dataSyncTransactionDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van-transactional + +## inventory transaction processed flag update URL, central, mmu +dataSyncProcessedFlagUpdate=${MMU_CENTRAL_SERVER}dataSync/updateProcessedFlagPostDownload + +## down-sync (central -> local) data API, central, mmu +downSyncDataUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van-downsync + +## down-sync (central -> local) flag update API, central, mmu +downSyncFlagUpdateUrl=${MMU_CENTRAL_SERVER}dataSync/updateDownSyncFlagPostDownload + +## mark an up-synced record as already delivered to the van (central only) +downSync.markDeliveredOnUpSync=true + + +fileBasePath =${MMU_FILE_BASE_PATH} + +## data sync batch size +BATCH_SIZE=30 +snomedCTPageSize=10 +nurseWL=7 +docWL=7 +pharmaWL=7 +labWL=7 +radioWL=7 +oncoWL=7 + +## For File Sync +serverIP=${FILE_SYNC_SERVER_IP} +serverDomain=${FILE_SYNC_SERVER_DOMAIN} +serverUserName=${FILE_SYNC_SERVER_USERNAME} +serverPassword=${FILE_SYNC_SERVER_PASSWORD} +localFolderToSync=${LOCAL_FOLDER_TO_SYNC} +serverFolder=MMUSync +getServerCredentialURL=${MMU_API}fileSyncController/getServerCredential +tmReferCheckValue=TM +### Redis IP +spring.redis.host=${REDIS_HOST} + +jwt.secret=${JWT_SECRET_KEY} + + +#ELK logging file name +logging.path=logs +logging.file.name=${MMU_API_LOGGING_FILE_NAME} + +springdoc.api-docs.enabled=${SWAGGER_DOC_ENABLED} springdoc.swagger-ui.enabled=${SWAGGER_DOC_ENABLED} \ No newline at end of file diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 69b99ba5..515bb23c 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -74,6 +74,15 @@ dataSyncTransactionDownloadUrl=http://10.208.122.38:8080/mmu-api/dataSync/server ## inventory transaction processed flag update URL, central, mmu dataSyncProcessedFlagUpdate=http://10.208.122.38:8080/mmu-api/dataSync/updateProcessedFlagPostDownload +## down-sync (central -> local) data API, central, mmu +downSyncDataUrl=http://10.208.122.38:8080/mmu-api/dataSync/server-to-van-downsync + +## down-sync (central -> local) flag update API, central, mmu +downSyncFlagUpdateUrl=http://10.208.122.38:8080/mmu-api/dataSync/updateDownSyncFlagPostDownload + +## mark an up-synced record as already delivered to the van (central only) +downSync.markDeliveredOnUpSync=true + ## fetching the TM case sheet for MMU. mmucentralserver = /mmu-api/common/get/Case-sheet/centralServerTMCaseSheet diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java new file mode 100644 index 00000000..c2f18050 --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -0,0 +1,185 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import com.google.gson.annotations.Expose; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.Transient; + +/*** + * Down-sync table configuration (central -> local). Analogous to + * {@link SyncUtilityClass} (m_synctabledetail) which drives the up-sync. + */ +@Entity +@Table(name = "m_downsynctabledetail") +public class DownSyncTableDetail { + + public static final String TABLE_TYPE_MASTER = "MASTER"; + public static final String TABLE_TYPE_TRANSACTIONAL = "TRANSACTIONAL"; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + @Expose + @Column(name = "DownSyncTableDetailID", updatable = false) + private Integer downSyncTableDetailID; + @Expose + @Column(name = "SchemaName") + private String schemaName; + @Expose + @Column(name = "TableName") + private String tableName; + /** columns to SELECT from central */ + @Expose + @Column(name = "ServerColumnName") + private String serverColumnName; + /** columns in local, positionally mapped with {@link #serverColumnName} */ + @Expose + @Column(name = "VanColumnName") + private String vanColumnName; + /** local auto-increment PK column - skipped on INSERT */ + @Expose + @Column(name = "VanAutoIncColumnName") + private String vanAutoIncColumnName; + /** MASTER = full pull, no VanID filter / TRANSACTIONAL = VanID + DownSynced filter */ + @Expose + @Column(name = "TableType") + private String tableType; + /** lower number syncs first - enforces the FK dependency chain */ + @Expose + @Column(name = "SyncOrder") + private Integer syncOrder; + @Expose + @Column(name = "IsActive") + private Boolean isActive; + + @Transient + @Expose + private Integer vanID; + @Transient + @Expose + private Integer providerServiceMapID; + + public DownSyncTableDetail() { + } + + public boolean isMasterTable() { + return tableType == null || TABLE_TYPE_MASTER.equalsIgnoreCase(tableType); + } + + public boolean isTransactionalTable() { + return TABLE_TYPE_TRANSACTIONAL.equalsIgnoreCase(tableType); + } + + public Integer getDownSyncTableDetailID() { + return downSyncTableDetailID; + } + + public void setDownSyncTableDetailID(Integer downSyncTableDetailID) { + this.downSyncTableDetailID = downSyncTableDetailID; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getServerColumnName() { + return serverColumnName; + } + + public void setServerColumnName(String serverColumnName) { + this.serverColumnName = serverColumnName; + } + + public String getVanColumnName() { + return vanColumnName; + } + + public void setVanColumnName(String vanColumnName) { + this.vanColumnName = vanColumnName; + } + + public String getVanAutoIncColumnName() { + return vanAutoIncColumnName; + } + + public void setVanAutoIncColumnName(String vanAutoIncColumnName) { + this.vanAutoIncColumnName = vanAutoIncColumnName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + + public Integer getSyncOrder() { + return syncOrder; + } + + public void setSyncOrder(Integer syncOrder) { + this.syncOrder = syncOrder; + } + + public Boolean getIsActive() { + return isActive; + } + + public void setIsActive(Boolean isActive) { + this.isActive = isActive; + } + + public Integer getVanID() { + return vanID; + } + + public void setVanID(Integer vanID) { + this.vanID = vanID; + } + + public Integer getProviderServiceMapID() { + return providerServiceMapID; + } + + public void setProviderServiceMapID(Integer providerServiceMapID) { + this.providerServiceMapID = providerServiceMapID; + } +} diff --git a/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java new file mode 100644 index 00000000..683bb10a --- /dev/null +++ b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/DownSyncTableDetailRepo.java @@ -0,0 +1,41 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.repo.syncActivity_syncLayer; + +import java.util.ArrayList; + +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; + +@Repository +public interface DownSyncTableDetailRepo extends CrudRepository { + + @Query(" SELECT d FROM DownSyncTableDetail d WHERE d.isActive = true ORDER BY d.syncOrder, d.downSyncTableDetailID ") + ArrayList getActiveDownSyncTables(); + + @Query(" SELECT d FROM DownSyncTableDetail d WHERE d.isActive = true AND lower(d.tableName) = lower(:tableName) ") + ArrayList getActiveDownSyncTableByName(@Param("tableName") String tableName); +} From ad2ccaa501e3634fe3e17ccff69270ec3a3d69cb Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Mon, 24 Aug 2026 16:47:08 +0530 Subject: [PATCH 39/81] fix: downsync implementation --- .../dataSyncActivity/StartSyncActivity.java | 49 ++ .../MMUDataSyncVanToServer.java | 52 ++ .../DownSyncDataDigester.java | 146 +++++ .../DownSyncRecordAck.java | 94 ++++ .../DownSyncTableDetail.java | 12 + .../dataSyncActivity/DataSyncRepository.java | 65 +++ .../DownSyncDataFromServer.java | 32 ++ .../DownSyncDataFromServerImpl.java | 522 ++++++++++++++++++ .../DataSyncRepositoryCentralDownload.java | 116 +++- .../GetDataFromVanAndSyncToDBImpl.java | 35 ++ .../GetDownSyncDataFromCentral.java | 35 ++ .../GetDownSyncDataFromCentralImpl.java | 125 +++++ 12 files changed, 1282 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 04f23f69..28b5a063 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -37,6 +37,7 @@ import org.springframework.web.bind.annotation.RestController; import com.google.gson.Gson; +import com.iemr.mmu.service.dataSyncActivity.DownSyncDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl; import com.iemr.mmu.service.dataSyncActivity.UploadDataToServerImpl; @@ -67,6 +68,8 @@ public class StartSyncActivity { private DiagnosticDocumentPushServiceImpl diagnosticDocumentPushServiceImpl; @Autowired private DiagnosticDocumentFetchService diagnosticDocumentFetchService; + @Autowired + private DownSyncDataFromServerImpl downSyncDataFromServerImpl; private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; @@ -273,4 +276,50 @@ public String downloadTransactionToLocal(@RequestBody String requestOBJ, return response.toString(); } + + @Operation(summary = "Start the down-sync of masters & transactional data from central to local") + @PostMapping(value = { "/startDownSync" }) + public String startDownSync(@RequestBody String requestOBJ, + @RequestHeader(value = "Authorization") String authorization, + @RequestHeader(value = "ServerAuthorization") String serverAuthorization, HttpServletRequest request) { + OutputResponse response = new OutputResponse(); + try { + String jwtToken = CookieUtil.getJwtTokenFromCookie(request); + + JSONObject obj = new JSONObject(requestOBJ); + if (obj.has("vanID") && obj.get("vanID") != null) { + Integer providerServiceMapID = obj.has(PROVIDER_SERVICE_MAP_ID) && !obj.isNull(PROVIDER_SERVICE_MAP_ID) + ? obj.getInt(PROVIDER_SERVICE_MAP_ID) + : null; + + String s = downSyncDataFromServerImpl.startDownSync(serverAuthorization, jwtToken, obj.getInt("vanID"), + providerServiceMapID); + + if (s != null && s.equalsIgnoreCase("inProgress")) + response.setError(5000, "Down-sync is already in progress, kindly wait for it to finish"); + else + response.setResponse(s); + } else { + response.setError(5000, "vanID is missing, Kindly contact the administrator."); + } + } catch (Exception e) { + logger.error("Error in down-sync : " + e); + response.setError(e); + } + return response.toString(); + } + + @Operation(summary = "Down-sync progress check") + @GetMapping(value = { "/checkDownSyncProgress" }) + public String checkDownSyncProgress() { + OutputResponse response = new OutputResponse(); + try { + response.setResponse(new Gson().toJson(downSyncDataFromServerImpl.getDownSyncStatus())); + } catch (Exception e) { + logger.error("Error in down-sync progress check : " + e); + response.setError(e); + } + return response.toString(); + } + } diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java index 4fd074c2..9accb7db 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java @@ -31,11 +31,13 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; import com.iemr.mmu.data.syncActivity_syncLayer.SyncDownloadMaster; import com.iemr.mmu.data.syncActivity_syncLayer.SyncUploadDataDigester; import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentIngestService; import com.iemr.mmu.service.dataSyncLayerCentral.FetchDownloadDataImpl; import com.iemr.mmu.service.dataSyncLayerCentral.GetDataFromVanAndSyncToDBImpl; +import com.iemr.mmu.service.dataSyncLayerCentral.GetDownSyncDataFromCentralImpl; import com.iemr.mmu.service.dataSyncLayerCentral.GetMasterDataFromCentralForVanImpl; import com.iemr.mmu.utils.response.OutputResponse; @@ -57,6 +59,8 @@ public class MMUDataSyncVanToServer { @Autowired private FetchDownloadDataImpl fetchDownloadDataImpl; @Autowired + private GetDownSyncDataFromCentralImpl getDownSyncDataFromCentralImpl; + @Autowired private DiagnosticDocumentIngestService diagnosticDocumentIngestService; @Operation(summary = "Sync data from van-to-server") @@ -175,4 +179,52 @@ public String updateProcessedFlagPostDownload(@RequestBody SyncUploadDataDigeste return response.toStringWithSerialization(); } + + @Operation(summary = "Down-sync data of one configured table from central to van") + @PostMapping(value = { "/server-to-van-downsync" }, consumes = "application/json", produces = "application/json") + public String downSyncDataFromServer(@RequestBody DownSyncDataDigester downSyncDataDigester, + @RequestHeader(value = "Authorization") String Authorization) { + OutputResponse response = new OutputResponse(); + try { + if (downSyncDataDigester != null) { + String s = getDownSyncDataFromCentralImpl.getDownSyncDataForVan(downSyncDataDigester); + if (s != null) + response.setResponse(s); + else + response.setError(5000, "Error in down-sync for table " + downSyncDataDigester.getSchemaName() + "." + + downSyncDataDigester.getTableName()); + } else { + response.setError(5000, "Invalid request"); + } + } catch (Exception e) { + if (downSyncDataDigester != null) + logger.error(e + " - Error in down-sync for table " + downSyncDataDigester.getSchemaName() + "." + + downSyncDataDigester.getTableName()); + response.setError(e); + } + return response.toStringWithSerialization(); + } + + @Operation(summary = "Update the down-sync flag at central post delivery to the van") + @PostMapping(value = { + "/updateDownSyncFlagPostDownload" }, consumes = "application/json", produces = "application/json") + public String updateDownSyncFlagPostDownload(@RequestBody DownSyncDataDigester downSyncDataDigester, + @RequestHeader(value = "Authorization") String Authorization) { + OutputResponse response = new OutputResponse(); + try { + if (downSyncDataDigester != null) { + int i = getDownSyncDataFromCentralImpl.updateDownSyncFlagPostDownload(downSyncDataDigester); + response.setResponse(String.valueOf(i)); + } else { + response.setError(5000, "Invalid request"); + } + } catch (Exception e) { + if (downSyncDataDigester != null) + logger.error(e + " - Error while updating the down-sync flag for " + + downSyncDataDigester.getSchemaName() + "." + downSyncDataDigester.getTableName()); + response.setError(e); + } + return response.toStringWithSerialization(); + } + } diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java new file mode 100644 index 00000000..0a9d8fc0 --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java @@ -0,0 +1,146 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import java.util.List; + +import com.google.gson.annotations.Expose; + +public class DownSyncDataDigester { + + @Expose + private String schemaName; + @Expose + private String tableName; + /** columns to SELECT from central */ + @Expose + private String serverColumnName; + /** primary key column of the table (same name in central & local) */ + @Expose + private String vanAutoIncColumnName; + /** MASTER / TRANSACTIONAL */ + @Expose + private String tableType; + @Expose + private Integer vanID; + @Expose + private Integer providerServiceMapID; + /** populated only on the acknowledgement call */ + @Expose + private List records; + + public DownSyncDataDigester() { + } + + public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, Integer vanID, + Integer providerServiceMapID) { + DownSyncDataDigester digester = new DownSyncDataDigester(); + digester.schemaName = tableDetail.getSchemaName(); + digester.tableName = tableDetail.getTableName(); + digester.serverColumnName = tableDetail.getServerColumnName(); + digester.vanAutoIncColumnName = tableDetail.getVanAutoIncColumnName(); + digester.tableType = tableDetail.getTableType(); + digester.vanID = vanID; + digester.providerServiceMapID = providerServiceMapID; + return digester; + } + + public static DownSyncDataDigester forAck(DownSyncTableDetail tableDetail, Integer vanID, + List records) { + DownSyncDataDigester digester = new DownSyncDataDigester(); + digester.schemaName = tableDetail.getSchemaName(); + digester.tableName = tableDetail.getTableName(); + digester.vanAutoIncColumnName = tableDetail.getVanAutoIncColumnName(); + digester.tableType = tableDetail.getTableType(); + digester.vanID = vanID; + digester.records = records; + return digester; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public String getServerColumnName() { + return serverColumnName; + } + + public void setServerColumnName(String serverColumnName) { + this.serverColumnName = serverColumnName; + } + + public String getVanAutoIncColumnName() { + return vanAutoIncColumnName; + } + + public void setVanAutoIncColumnName(String vanAutoIncColumnName) { + this.vanAutoIncColumnName = vanAutoIncColumnName; + } + + public String getTableType() { + return tableType; + } + + public void setTableType(String tableType) { + this.tableType = tableType; + } + + public Integer getVanID() { + return vanID; + } + + public void setVanID(Integer vanID) { + this.vanID = vanID; + } + + public Integer getProviderServiceMapID() { + return providerServiceMapID; + } + + public void setProviderServiceMapID(Integer providerServiceMapID) { + this.providerServiceMapID = providerServiceMapID; + } + + public List getRecords() { + return records; + } + + public void setRecords(List records) { + this.records = records; + } + + public boolean isMasterTable() { + return tableType == null || DownSyncTableDetail.TABLE_TYPE_MASTER.equalsIgnoreCase(tableType); + } +} diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java new file mode 100644 index 00000000..5be35f81 --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java @@ -0,0 +1,94 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import com.google.gson.annotations.Expose; + +public class DownSyncRecordAck { + + public static final String STATUS_PROCESSED = "P"; + public static final String STATUS_FAILED = "F"; + public static final String CONFLICT = "CONFLICT"; + + @Expose + private Long centralID; + @Expose + private Long vanSerialNo; + @Expose + private String status; + @Expose + private String failureReason; + + public DownSyncRecordAck() { + } + + public DownSyncRecordAck(Long centralID, Long vanSerialNo, String status, String failureReason) { + this.centralID = centralID; + this.vanSerialNo = vanSerialNo; + this.status = status; + this.failureReason = failureReason; + } + + public static DownSyncRecordAck success(Long centralID, Long vanSerialNo) { + return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_PROCESSED, null); + } + + public static DownSyncRecordAck failure(Long centralID, Long vanSerialNo, String failureReason) { + return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_FAILED, failureReason); + } + + public boolean isSuccess() { + return STATUS_PROCESSED.equalsIgnoreCase(status); + } + + public Long getCentralID() { + return centralID; + } + + public void setCentralID(Long centralID) { + this.centralID = centralID; + } + + public Long getVanSerialNo() { + return vanSerialNo; + } + + public void setVanSerialNo(Long vanSerialNo) { + this.vanSerialNo = vanSerialNo; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getFailureReason() { + return failureReason; + } + + public void setFailureReason(String failureReason) { + this.failureReason = failureReason; + } +} diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index c2f18050..85e70fe6 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -69,6 +69,9 @@ public class DownSyncTableDetail { @Expose @Column(name = "TableType") private String tableType; + + @Column(name = "LastModColumnName") + private String lastModColumnName; /** lower number syncs first - enforces the FK dependency chain */ @Expose @Column(name = "SyncOrder") @@ -135,6 +138,15 @@ public void setVanColumnName(String vanColumnName) { this.vanColumnName = vanColumnName; } + public String getLastModColumnName() { + return (lastModColumnName == null || lastModColumnName.trim().isEmpty()) ? "LastModDate" + : lastModColumnName.trim(); + } + + public void setLastModColumnName(String lastModColumnName) { + this.lastModColumnName = lastModColumnName; + } + public String getVanAutoIncColumnName() { return vanAutoIncColumnName; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index dc36c7c1..6bbc33f8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -26,6 +26,8 @@ import java.util.List; import java.util.Map; +import java.sql.PreparedStatement; + import javax.sql.DataSource; import org.slf4j.Logger; @@ -33,6 +35,8 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.annotation.CreatedDate; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.GeneratedKeyHolder; +import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Service; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo; @@ -137,4 +141,65 @@ public int[] updateLatestMasterInLocal(String query, List syncDataList // ---------------------------------- End of Download Repository + // ---------------------------------- Down-Sync Repository (central -> local) + + public List getDownSyncColumns(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + + " AND COLUMN_NAME NOT IN ('DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " + + " 'LastDownSyncDate') " + " ORDER BY COLUMN_NAME "; + + return jdbcTemplate.queryForList(query, String.class, schema, table); + } + + public Map getLocalRecordForDownSync(String schema, String table, String autoIncColumnName, + Object vanSerialNo, Object vanID, String lastModColumn) { + if (vanSerialNo == null) + return null; + + jdbcTemplate = getJdbcTemplate(); + String query = " SELECT " + autoIncColumnName + ", Processed, " + lastModColumn + + " AS LastModDate, LastDownSyncDate FROM " + schema + "." + table + + " WHERE " + autoIncColumnName + " = ? AND VanID = ? "; + + List> resultSet = jdbcTemplate.queryForList(query, vanSerialNo, vanID); + if (resultSet == null || resultSet.isEmpty()) + return null; + + return resultSet.get(0); + } + + public Long insertDownSyncRecordInLocal(final String query, final Object[] params) { + jdbcTemplate = getJdbcTemplate(); + KeyHolder keyHolder = new GeneratedKeyHolder(); + + jdbcTemplate.update(connection -> { + PreparedStatement ps = connection.prepareStatement(query, PreparedStatement.RETURN_GENERATED_KEYS); + for (int i = 0; i < params.length; i++) { + ps.setObject(i + 1, params[i]); + } + return ps; + }, keyHolder); + + Number key = keyHolder.getKey(); + return key != null ? key.longValue() : null; + } + + public int updateDownSyncRecordInLocal(String query, Object[] params) { + jdbcTemplate = getJdbcTemplate(); + return jdbcTemplate.update(query, params); + } + + public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { + jdbcTemplate = getJdbcTemplate(); + String query = " UPDATE " + schema + "." + table + + " SET Processed = 'F', SyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; + return jdbcTemplate.update(query, localID); + } + + // End of Down-Sync Repository + + } \ No newline at end of file diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java new file mode 100644 index 00000000..3fcabcf8 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServer.java @@ -0,0 +1,32 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import java.util.Map; + +public interface DownSyncDataFromServer { + + String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, Integer providerServiceMapID) + throws Exception; + + Map getDownSyncStatus(); +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java new file mode 100644 index 00000000..632af829 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -0,0 +1,522 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import java.sql.Timestamp; +import java.text.DateFormat; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; +import com.iemr.mmu.repo.syncActivity_syncLayer.DownSyncTableDetailRepo; +import com.iemr.mmu.utils.RestTemplateUtil; + + +@Service +@PropertySource("classpath:application.properties") +public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { + + private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + private static final String VAN_SERIAL_NO = "VanSerialNo"; + private static final String VAN_ID = "VanID"; + private static final String LAST_MOD_DATE = "LastModDate"; + private static final String LAST_DOWN_SYNC_DATE = "LastDownSyncDate"; + private static final String DOWN_SYNCED = "DownSynced"; + private static final String DOWN_SYNC_DATE = "DownSyncDate"; + private static final String DOWN_SYNC_FAILURE_REASON = "DownSyncFailureReason"; + private static final String PROCESSED = "Processed"; + private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; + + @Value("${downSyncDataUrl}") + private String downSyncDataUrl; + + @Value("${downSyncFlagUpdateUrl}") + private String downSyncFlagUpdateUrl; + + @Autowired + private DownSyncTableDetailRepo downSyncTableDetailRepo; + + @Autowired + private DataSyncRepository dataSyncRepository; + + private static final AtomicBoolean IN_PROGRESS = new AtomicBoolean(false); + private static int totalCounter = 0; + private static int progressCounter = 0; + private static int insertedCounter = 0; + private static int updatedCounter = 0; + private static int conflictCounter = 0; + private static int skippedCounter = 0; + private static int failedCounter = 0; + private static StringBuilder failedTables = new StringBuilder(); + private static String currentTable = ""; + + public String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, + Integer providerServiceMapID) throws Exception { + + if (vanID == null) + throw new Exception("vanID is mandatory for down-sync. Kindly contact the administrator."); + + final ArrayList downSyncTables = downSyncTableDetailRepo.getActiveDownSyncTables(); + if (downSyncTables == null || downSyncTables.isEmpty()) + throw new Exception("No table is configured for down-sync in m_downsynctabledetail."); + + if (!IN_PROGRESS.compareAndSet(false, true)) + return "inProgress"; + + totalCounter = downSyncTables.size(); + progressCounter = 0; + insertedCounter = 0; + updatedCounter = 0; + conflictCounter = 0; + skippedCounter = 0; + failedCounter = 0; + failedTables = new StringBuilder(); + currentTable = ""; + + final ExecutorService threadPool = Executors.newSingleThreadExecutor(); + try { + threadPool.submit(() -> { + try { + for (DownSyncTableDetail tableDetail : downSyncTables) { + currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); + try { + downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); + } catch (Exception e) { + failedCounter++; + failedTables.append(tableDetail.getTableName()).append(" | "); + logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); + } + progressCounter++; + } + } finally { + IN_PROGRESS.set(false); + } + return "Down-sync completed"; + }); + } finally { + threadPool.shutdown(); + } + + return " Down-sync started "; + } + + public Map getDownSyncStatus() { + Map resultMap = new LinkedHashMap<>(); + resultMap.put("percentage", totalCounter == 0 ? 0 : Math.floor((progressCounter * 100) / totalCounter)); + resultMap.put("inProgress", IN_PROGRESS.get()); + resultMap.put("currentTable", currentTable); + resultMap.put("totalTables", totalCounter); + resultMap.put("completedTables", progressCounter); + resultMap.put("failedTableCount", failedCounter); + resultMap.put("failedTables", failedTables.toString()); + resultMap.put("recordsInserted", insertedCounter); + resultMap.put("recordsUpdated", updatedCounter); + resultMap.put("recordsSkipped", skippedCounter); + resultMap.put("conflicts", conflictCounter); + return resultMap; + } + + /*** + * Down-syncs one configured table. + */ + private void downSyncTable(DownSyncTableDetail tableDetail, Integer vanID, Integer providerServiceMapID, + String serverAuthorization, String jwtToken) throws Exception { + + List serverColumns = splitColumns(tableDetail.getServerColumnName()); + List vanColumns = splitColumns(tableDetail.getVanColumnName()); + + if (serverColumns.isEmpty() && vanColumns.isEmpty()) { + List resolved = dataSyncRepository.getDownSyncColumns(tableDetail.getSchemaName(), + tableDetail.getTableName()); + if (resolved.isEmpty()) + throw new Exception("No column found locally for " + tableDetail.getSchemaName() + "." + + tableDetail.getTableName() + + ". The table is configured for down-sync but does not exist in the local DB."); + + serverColumns = resolved; + vanColumns = resolved; + } + + if (serverColumns.isEmpty() || vanColumns.isEmpty()) + throw new Exception("Only one of ServerColumnName / VanColumnName is configured for " + + tableDetail.getTableName() + ". Configure both, with the same columns in the same order, " + + "or leave both empty to have them resolved from the schema."); + if (serverColumns.size() != vanColumns.size()) + throw new Exception("ServerColumnName & VanColumnName column count does not match for " + + tableDetail.getTableName() + " (" + serverColumns.size() + " vs " + vanColumns.size() + + "). Central and local must expose the same number of columns, in the same order."); + + List> dataFromCentral = fetchDataFromCentral(tableDetail, String.join(",", serverColumns), + vanID, providerServiceMapID, serverAuthorization, jwtToken); + + if (dataFromCentral.isEmpty()) { + logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), tableDetail.getTableName()); + return; + } + + if (tableDetail.isMasterTable()) { + upsertMasterData(tableDetail, serverColumns, vanColumns, dataFromCentral); + } else { + List acks = saveTransactionalData(tableDetail, serverColumns, vanColumns, + dataFromCentral, vanID); + acknowledgeToCentral(tableDetail, vanID, acks, serverAuthorization, jwtToken); + } + } + + private List> fetchDataFromCentral(DownSyncTableDetail tableDetail, String serverColumnName, + Integer vanID, Integer providerServiceMapID, String serverAuthorization, String jwtToken) + throws Exception { + + DownSyncDataDigester digester = DownSyncDataDigester.forDownload(tableDetail, serverColumnName, vanID, + providerServiceMapID); + + RestTemplate restTemplate = new RestTemplate(); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + ResponseEntity response = restTemplate.exchange(downSyncDataUrl, HttpMethod.POST, request, String.class); + + if (response == null || !response.hasBody()) + throw new Exception("Empty response from central for " + tableDetail.getTableName()); + + JSONObject responseObj = new JSONObject(response.getBody()); + if (!responseObj.has("data") || !responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) + throw new Exception("Down-sync API failed for " + tableDetail.getTableName() + " : " + + responseObj.optString("errorMessage")); + + Gson gson = new GsonBuilder().serializeNulls().create(); + List> data = gson.fromJson(responseObj.get("data").toString(), + new TypeToken>>() { + }.getType()); + + return data != null ? data : new ArrayList<>(); + } + + /*** + * MASTER tables : central authoritative, upserted in one batch. + */ + private void upsertMasterData(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, + List> dataFromCentral) { + + StringBuilder placeHolders = new StringBuilder(); + StringBuilder onDuplicate = new StringBuilder(); + for (int i = 0; i < vanColumns.size(); i++) { + if (i > 0) { + placeHolders.append(", "); + onDuplicate.append(", "); + } + placeHolders.append("?"); + onDuplicate.append(vanColumns.get(i)).append(" = VALUES(").append(vanColumns.get(i)).append(")"); + } + + String query = " INSERT INTO " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " ( " + + String.join(", ", vanColumns) + " ) VALUES ( " + placeHolders + " ) ON DUPLICATE KEY UPDATE " + + onDuplicate; + + List batch = new ArrayList<>(); + for (Map record : dataFromCentral) { + Object[] values = new Object[serverColumns.size()]; + for (int i = 0; i < serverColumns.size(); i++) { + values[i] = normalize(record.get(serverColumns.get(i))); + } + batch.add(values); + } + + int[] result = dataSyncRepository.updateLatestMasterInLocal(query, batch); + updatedCounter += result != null ? result.length : 0; + logger.info("Down-synced {} master records into {}.{}", batch.size(), tableDetail.getSchemaName(), + tableDetail.getTableName()); + } + + /*** + * TRANSACTIONAL tables : record by record, so that a conflict on one record + * does not block the delivery of the rest. + */ + private List saveTransactionalData(DownSyncTableDetail tableDetail, List serverColumns, + List vanColumns, List> dataFromCentral, Integer vanID) throws Exception { + + String pkColumn = tableDetail.getVanAutoIncColumnName(); + if (pkColumn == null || pkColumn.trim().isEmpty()) + throw new Exception( + "VanAutoIncColumnName is not configured for transactional table " + tableDetail.getTableName()); + pkColumn = pkColumn.trim(); + + requireIdentifier(pkColumn, "VanAutoIncColumnName", tableDetail); + requireIdentifier(tableDetail.getLastModColumnName(), "LastModColumnName", tableDetail); + + List acks = new ArrayList<>(); + + for (Map record : dataFromCentral) { + Long centralID = toLong(record.get(pkColumn)); + Object vanSerialNo = normalize(record.get(VAN_SERIAL_NO)); + + try { + Map localRecord = dataSyncRepository.getLocalRecordForDownSync( + tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, vanSerialNo, vanID, + tableDetail.getLastModColumnName()); + + if (localRecord == null) { + Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); + insertedCounter++; + acks.add(DownSyncRecordAck.success(centralID, localID)); + continue; + } + + Long localID = toLong(localRecord.get(pkColumn)); + + if (!isCentralCopyNewer(record, localRecord, tableDetail.getLastModColumnName())) { + skippedCounter++; + acks.add(DownSyncRecordAck.success(centralID, localID)); + continue; + } + + if (hasUnsentLocalChanges(localRecord)) { + dataSyncRepository.markDownSyncConflictInLocal(tableDetail.getSchemaName(), + tableDetail.getTableName(), pkColumn, localID); + conflictCounter++; + acks.add(DownSyncRecordAck.failure(centralID, localID, DownSyncRecordAck.CONFLICT)); + logger.warn("Down-sync conflict for {}.{}, local id {}", tableDetail.getSchemaName(), + tableDetail.getTableName(), localID); + continue; + } + + updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, localID); + updatedCounter++; + acks.add(DownSyncRecordAck.success(centralID, localID)); + + } catch (Exception e) { + failedCounter++; + String reason = e.getMessage() != null && e.getMessage().length() > 250 + ? e.getMessage().substring(0, 250) + : e.getMessage(); + acks.add(DownSyncRecordAck.failure(centralID, toLong(vanSerialNo), reason)); + logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + + e.getMessage(), e); + } + } + + return acks; + } + + private boolean isCentralCopyNewer(Map centralRecord, Map localRecord, + String lastModColumn) { + Timestamp centralLastModDate = toTimestamp(centralRecord.get(lastModColumn)); + if (centralLastModDate == null) + return false; + + Timestamp localLastModDate = toTimestamp(localRecord.get(LAST_MOD_DATE)); + if (localLastModDate != null) + return centralLastModDate.after(localLastModDate); + + Timestamp localLastDownSyncDate = toTimestamp(localRecord.get(LAST_DOWN_SYNC_DATE)); + return localLastDownSyncDate == null || centralLastModDate.after(localLastDownSyncDate); + } + + private boolean hasUnsentLocalChanges(Map localRecord) { + String localProcessed = localRecord.get(PROCESSED) != null ? String.valueOf(localRecord.get(PROCESSED)) : null; + return "N".equalsIgnoreCase(localProcessed); + } + + private Long insertRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, + String pkColumn, Map record) { + + List columns = new ArrayList<>(); + List values = new ArrayList<>(); + + for (int i = 0; i < vanColumns.size(); i++) { + String vanColumn = vanColumns.get(i); + // the local auto-increment PK is skipped so that local generates its own value + if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + continue; + + columns.add(vanColumn); + values.add(normalize(record.get(serverColumns.get(i)))); + } + + columns.add(PROCESSED); + values.add("P"); + columns.add(SYNC_FAILURE_REASON); + values.add(null); + columns.add(LAST_DOWN_SYNC_DATE); + values.add(Timestamp.valueOf(LocalDateTime.now())); + + StringBuilder placeHolders = new StringBuilder(); + for (int i = 0; i < columns.size(); i++) { + if (i > 0) + placeHolders.append(", "); + placeHolders.append("?"); + } + + String query = " INSERT INTO " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " ( " + + String.join(", ", columns) + " ) VALUES ( " + placeHolders + " ) "; + + return dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + } + + private void updateRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, + String pkColumn, Map record, Long localID) { + + StringBuilder setClause = new StringBuilder(); + List values = new ArrayList<>(); + + for (int i = 0; i < vanColumns.size(); i++) { + String vanColumn = vanColumns.get(i); + if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + continue; + + if (setClause.length() > 0) + setClause.append(", "); + setClause.append(vanColumn).append(" = ?"); + values.add(normalize(record.get(serverColumns.get(i)))); + } + + setClause.append(setClause.length() > 0 ? ", " : "").append(PROCESSED).append(" = 'P', ") + .append(SYNC_FAILURE_REASON).append(" = NULL, ").append(LAST_DOWN_SYNC_DATE).append(" = ? "); + values.add(Timestamp.valueOf(LocalDateTime.now())); + values.add(localID); + + String query = " UPDATE " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " SET " + setClause + + " WHERE " + pkColumn + " = ? "; + + dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); + } + + private void requireIdentifier(String value, String what, DownSyncTableDetail tableDetail) throws Exception { + if (value == null || !value.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + throw new Exception("The down-sync configuration of " + tableDetail.getTableName() + " holds an invalid " + + what); + } + + private boolean isDownSyncManagedColumn(String column) { + return PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) + || LAST_DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNCED.equalsIgnoreCase(column) + || DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNC_FAILURE_REASON.equalsIgnoreCase(column); + } + + private void acknowledgeToCentral(DownSyncTableDetail tableDetail, Integer vanID, List acks, + String serverAuthorization, String jwtToken) throws Exception { + + if (acks == null || acks.isEmpty()) + return; + + DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, acks); + + RestTemplate restTemplate = new RestTemplate(); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + ResponseEntity response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, + String.class); + + if (response == null || !response.hasBody()) + throw new Exception("Empty response while updating the down-sync flag for " + tableDetail.getTableName()); + + JSONObject responseObj = new JSONObject(response.getBody()); + if (!responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) + throw new Exception("Could not update the down-sync flag at central for " + tableDetail.getTableName() + + " : " + responseObj.optString("errorMessage")); + + logger.info("Down-sync flag updated at central for {} records of {}", acks.size(), tableDetail.getTableName()); + } + + private List splitColumns(String columns) { + List columnList = new ArrayList<>(); + if (columns == null) + return columnList; + + for (String column : Arrays.asList(columns.split(","))) { + if (column != null && !column.trim().isEmpty()) + columnList.add(column.trim()); + } + return columnList; + } + + private Object normalize(Object value) { + if (value instanceof Double) { + double doubleValue = (Double) value; + if (!Double.isInfinite(doubleValue) && !Double.isNaN(doubleValue) && doubleValue == Math.floor(doubleValue) + && Math.abs(doubleValue) < 1e15) + return (long) doubleValue; + } + return value; + } + + private Long toLong(Object value) { + Object normalized = normalize(value); + if (normalized == null) + return null; + if (normalized instanceof Number) + return ((Number) normalized).longValue(); + try { + return Long.valueOf(String.valueOf(normalized).trim()); + } catch (NumberFormatException e) { + return null; + } + } + + private Timestamp toTimestamp(Object value) { + if (value == null) + return null; + if (value instanceof Timestamp) + return (Timestamp) value; + if (value instanceof java.util.Date) + return new Timestamp(((java.util.Date) value).getTime()); + + String stringValue = String.valueOf(value).trim(); + if (stringValue.isEmpty() || "null".equalsIgnoreCase(stringValue)) + return null; + + String candidate = stringValue.replace("T", " ").replace("Z", ""); + try { + return Timestamp.valueOf(candidate); + } catch (IllegalArgumentException e) { + try { + // gson's default rendering of a java.sql.Timestamp + return new Timestamp(DateFormat.getDateTimeInstance().parse(stringValue).getTime()); + } catch (java.text.ParseException pe) { + logger.warn("Could not parse the timestamp '{}' received from central", stringValue); + return null; + } + } + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 565e2466..2cceb6d8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -28,6 +28,8 @@ import javax.sql.DataSource; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -54,6 +56,8 @@ private JdbcTemplate getJdbcTemplate() { private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private static final String DOWN_SYNC_TRANSACTIONAL = "TRANSACTIONAL"; + // Data Upload Repository public int checkRecordIsAlreadyPresentOrNot(String schemaName, String tableName, String vanSerialNo, String vanID, String vanAutoIncColumnName, int syncFacilityID) { @@ -205,4 +209,114 @@ else if (masterType.equalsIgnoreCase("P")) { } // End of Data Download Repository -} \ No newline at end of file + + // ---------------------------------- Down-Sync Repository (central -> local) + + public List> getDownSyncDataFromTable(String schema, String table, String columnNames, + String tableType, Integer vanID, String lastModColumn) throws Exception { + jdbcTemplate = getJdbcTemplate(); + + if (schema == null || table == null) + throw new Exception("Invalid down-sync request. Schema/table info is missing"); + + String columns = (columnNames == null || columnNames.trim().isEmpty()) ? " * " : columnNames; + List> resultSetList; + + if (DOWN_SYNC_TRANSACTIONAL.equalsIgnoreCase(tableType)) { + if (vanID == null) + throw new Exception("Invalid down-sync request. VanID is mandatory for transactional table " + table); + + String query = " SELECT " + columns + " FROM " + schema + "." + table + + " WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') " + + " OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND " + lastModColumn + + " > DownSyncDate ) ) "; + logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + resultSetList = jdbcTemplate.queryForList(query, vanID); + } else { + String query = " SELECT " + columns + " FROM " + schema + "." + table; + logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + resultSetList = jdbcTemplate.queryForList(query); + } + + logger.info("Down-sync record count for {}.{} : {}", schema, table, resultSetList.size()); + return resultSetList; + } + + public int updateDownSyncFlagPostDownload(String schema, String table, String pkColumnName, + List records) throws Exception { + jdbcTemplate = getJdbcTemplate(); + + if (schema == null || table == null || pkColumnName == null) + throw new Exception("Invalid request. Schema/table/primary key info is missing"); + if (records == null || records.isEmpty()) + return 0; + + List successWithSerialNo = new ArrayList<>(); + List successWithoutSerialNo = new ArrayList<>(); + List failed = new ArrayList<>(); + + for (DownSyncRecordAck record : records) { + if (record == null || record.getCentralID() == null) + continue; + + if (record.isSuccess()) { + if (record.getVanSerialNo() != null) + successWithSerialNo.add(new Object[] { record.getVanSerialNo(), record.getCentralID() }); + else + successWithoutSerialNo.add(new Object[] { record.getCentralID() }); + } else { + failed.add(new Object[] { record.getFailureReason(), record.getCentralID() }); + } + } + + int updatedRows = 0; + + if (!successWithSerialNo.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL, VanSerialNo = ? " + + " WHERE " + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithSerialNo)); + } + + if (!successWithoutSerialNo.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE " + + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); + } + + if (!failed.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, failed)); + } + + logger.info("Down-sync flag updated for {}.{}. Records : {}, rows : {}", schema, table, records.size(), + updatedRows); + return updatedRows; + } + + public int markDownSyncedPostUpSync(String schema, String table, List vanSerialNoAndVanID) { + if (schema == null || table == null || vanSerialNoAndVanID == null || vanSerialNoAndVanID.isEmpty()) + return 0; + + jdbcTemplate = getJdbcTemplate(); + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + + " WHERE VanSerialNo = ? AND VanID = ? "; + return countUpdatedRows(jdbcTemplate.batchUpdate(query, vanSerialNoAndVanID)); + } + + private int countUpdatedRows(int[] batchResult) { + int count = 0; + if (batchResult != null) { + for (int rows : batchResult) { + if (rows > 0) + count += rows; + } + } + return count; + } + + // End of Down-Sync Repository +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java index cca5634d..3205007c 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java @@ -31,6 +31,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import com.fasterxml.jackson.databind.ObjectMapper; @@ -46,6 +47,12 @@ public class GetDataFromVanAndSyncToDBImpl implements GetDataFromVanAndSyncToDB @Autowired private DataSyncRepositoryCentral dataSyncRepositoryCentral; + @Autowired + private GetDownSyncDataFromCentral getDownSyncDataFromCentral; + + @Value("${downSync.markDeliveredOnUpSync:true}") + private boolean markDownSyncedOnUpSync; + private static final Map> TABLE_GROUPS = new HashMap<>(); static { TABLE_GROUPS.put(1, @@ -326,6 +333,7 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig Map insertIndexMap = new HashMap<>(); Map updateIndexMap = new HashMap<>(); + Map vanIDByResultIndex = new HashMap<>(); boolean overallSuccess = true; @@ -463,6 +471,7 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig int currentSyncResultIndex = syncResults.size(); syncResults.add(new SyncResult(schemaName, syncTableName, vanSerialNo, syncUploadDataDigester.getSyncedBy(), false, "Pending")); + vanIDByResultIndex.put(currentSyncResultIndex, vanID); if (recordCheck == 0) { insertIndexMap.put(currentSyncResultIndex, syncDataListInsert.size()); @@ -591,9 +600,35 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig } logger.info("Sync results for table {}: {}", syncTableName, syncResults); + + flagUpSyncedRecordsForDownSync(schemaName, syncTableName, syncResults, vanIDByResultIndex); + return overallSuccess; } + private void flagUpSyncedRecordsForDownSync(String schemaName, String tableName, List syncResults, + Map vanIDByResultIndex) { + if (!markDownSyncedOnUpSync || vanIDByResultIndex.isEmpty()) + return; + + List vanSerialNoAndVanID = new ArrayList<>(); + for (Map.Entry entry : vanIDByResultIndex.entrySet()) { + int index = entry.getKey(); + if (index >= syncResults.size()) + continue; + + SyncResult result = syncResults.get(index); + if (result == null || !result.isSuccess() || result.getVanSerialNo() == null + || entry.getValue() == null || "null".equalsIgnoreCase(entry.getValue())) + continue; + + vanSerialNoAndVanID.add(new Object[] { result.getVanSerialNo(), entry.getValue() }); + } + + if (!vanSerialNoAndVanID.isEmpty()) + getDownSyncDataFromCentral.markDownSyncedPostUpSync(schemaName, tableName, vanSerialNoAndVanID); + } + private String getVanSerialNo(Object[] record, int vanSerialIndex, SyncResult originalResult) { if (vanSerialIndex >= 0 && vanSerialIndex < record.length) { return String.valueOf(record[vanSerialIndex]); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java new file mode 100644 index 00000000..553ee73e --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentral.java @@ -0,0 +1,35 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.util.List; + +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; + +public interface GetDownSyncDataFromCentral { + + String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) throws Exception; + + int updateDownSyncFlagPostDownload(DownSyncDataDigester downSyncDataDigester) throws Exception; + + int markDownSyncedPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID); +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java new file mode 100644 index 00000000..42e71dc6 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -0,0 +1,125 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; +import com.iemr.mmu.repo.syncActivity_syncLayer.DownSyncTableDetailRepo; + + +@Service +public class GetDownSyncDataFromCentralImpl implements GetDownSyncDataFromCentral { + + private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + @Autowired + private DataSyncRepositoryCentralDownload dataSyncRepositoryCentralDownload; + + @Autowired + private DownSyncTableDetailRepo downSyncTableDetailRepo; + + public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) throws Exception { + if (downSyncDataDigester == null || downSyncDataDigester.getSchemaName() == null + || downSyncDataDigester.getTableName() == null) + throw new Exception("Invalid down-sync request. Either schema or table info is missing/wrong"); + + if (!downSyncDataDigester.isMasterTable() && downSyncDataDigester.getVanID() == null) + throw new Exception("Invalid down-sync request. VanID is mandatory for a transactional table"); + + List> resultSetList = dataSyncRepositoryCentralDownload.getDownSyncDataFromTable( + downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), + downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), + downSyncDataDigester.getVanID()); + + return downSyncGson().toJson(resultSetList); + } + + + private Gson downSyncGson() { + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.serializeNulls(); + gsonBuilder.registerTypeAdapter(Timestamp.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive(src.toString())); + gsonBuilder.registerTypeAdapter(java.sql.Date.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive(src.toString())); + gsonBuilder.registerTypeAdapter(java.sql.Time.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive(src.toString())); + gsonBuilder.registerTypeAdapter(LocalDateTime.class, + (JsonSerializer) (src, type, context) -> new JsonPrimitive( + Timestamp.valueOf(src).toString())); + return gsonBuilder.create(); + } + + public int updateDownSyncFlagPostDownload(DownSyncDataDigester downSyncDataDigester) throws Exception { + if (downSyncDataDigester == null || downSyncDataDigester.getSchemaName() == null + || downSyncDataDigester.getTableName() == null || downSyncDataDigester.getRecords() == null) + throw new Exception("Invalid request. Schema, table or record info is missing/wrong"); + + String pkColumnName = downSyncDataDigester.getVanAutoIncColumnName(); + if (pkColumnName == null || pkColumnName.trim().isEmpty()) + throw new Exception("Invalid request. Primary key column of " + downSyncDataDigester.getTableName() + + " is missing in the request"); + + return dataSyncRepositoryCentralDownload.updateDownSyncFlagPostDownload(downSyncDataDigester.getSchemaName(), + downSyncDataDigester.getTableName(), pkColumnName.trim(), downSyncDataDigester.getRecords()); + } + + + public int resetDownSyncFlagPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID) { + if (schemaName == null || tableName == null || vanSerialNoAndVanID == null || vanSerialNoAndVanID.isEmpty()) + return 0; + + try { + ArrayList tableDetails = downSyncTableDetailRepo + .getActiveDownSyncTableByName(tableName); + boolean downSyncEnabled = false; + for (DownSyncTableDetail tableDetail : tableDetails) { + if (tableDetail.isTransactionalTable()) + downSyncEnabled = true; + } + if (!downSyncEnabled) + return 0; + + return dataSyncRepositoryCentralDownload.resetDownSyncFlagPostUpSync(schemaName, tableName, + vanSerialNoAndVanID); + } catch (Exception e) { + logger.warn("Could not reset the DownSynced flag for {}.{} post up-sync : {}", schemaName, tableName, + e.getMessage()); + return 0; + } + } +} From b0ca63fd70589852467c7368f4b8f3acfba47583 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Mon, 24 Aug 2026 17:01:18 +0530 Subject: [PATCH 40/81] fix: build issue --- .../DownSyncDataDigester.java | 6 ++-- .../GetDownSyncDataFromCentralImpl.java | 28 +++++++++++++++---- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java index 0a9d8fc0..af143161 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java @@ -51,12 +51,12 @@ public class DownSyncDataDigester { public DownSyncDataDigester() { } - public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, Integer vanID, - Integer providerServiceMapID) { + public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, String serverColumnName, + Integer vanID, Integer providerServiceMapID) { DownSyncDataDigester digester = new DownSyncDataDigester(); digester.schemaName = tableDetail.getSchemaName(); digester.tableName = tableDetail.getTableName(); - digester.serverColumnName = tableDetail.getServerColumnName(); + digester.serverColumnName = serverColumnName; digester.vanAutoIncColumnName = tableDetail.getVanAutoIncColumnName(); digester.tableType = tableDetail.getTableType(); digester.vanID = vanID; diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java index 42e71dc6..54b931ac 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -63,12 +63,28 @@ public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) t List> resultSetList = dataSyncRepositoryCentralDownload.getDownSyncDataFromTable( downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), - downSyncDataDigester.getVanID()); + downSyncDataDigester.getVanID(), resolveLastModColumn(downSyncDataDigester.getTableName())); return downSyncGson().toJson(resultSetList); } - + private String resolveLastModColumn(String tableName) throws Exception { + String lastModColumn = "LastModDate"; + + if (tableName != null) { + ArrayList tableDetails = downSyncTableDetailRepo + .getActiveDownSyncTableByName(tableName.trim()); + if (tableDetails != null && !tableDetails.isEmpty()) + lastModColumn = tableDetails.get(0).getLastModColumnName(); + } + + if (lastModColumn == null || !lastModColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + throw new Exception( + "The down-sync configuration of " + tableName + " holds an invalid modification-time column name"); + + return lastModColumn; + } + private Gson downSyncGson() { GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.serializeNulls(); @@ -98,8 +114,8 @@ public int updateDownSyncFlagPostDownload(DownSyncDataDigester downSyncDataDiges downSyncDataDigester.getTableName(), pkColumnName.trim(), downSyncDataDigester.getRecords()); } - - public int resetDownSyncFlagPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID) { + @Override + public int markDownSyncedPostUpSync(String schemaName, String tableName, List vanSerialNoAndVanID) { if (schemaName == null || tableName == null || vanSerialNoAndVanID == null || vanSerialNoAndVanID.isEmpty()) return 0; @@ -114,10 +130,10 @@ public int resetDownSyncFlagPostUpSync(String schemaName, String tableName, List if (!downSyncEnabled) return 0; - return dataSyncRepositoryCentralDownload.resetDownSyncFlagPostUpSync(schemaName, tableName, + return dataSyncRepositoryCentralDownload.markDownSyncedPostUpSync(schemaName, tableName, vanSerialNoAndVanID); } catch (Exception e) { - logger.warn("Could not reset the DownSynced flag for {}.{} post up-sync : {}", schemaName, tableName, + logger.warn("Could not mark the DownSynced flag for {}.{} post up-sync : {}", schemaName, tableName, e.getMessage()); return 0; } From e9ddd75d2c802fcfd226b8120dfa2c4d83f26bd4 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 25 Aug 2026 20:08:23 +0530 Subject: [PATCH 41/81] fix: lastmoddate updation --- .../syncActivity_syncLayer/DownSyncTableDetail.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index 85e70fe6..c7a0dd68 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -41,6 +41,7 @@ public class DownSyncTableDetail { public static final String TABLE_TYPE_MASTER = "MASTER"; public static final String TABLE_TYPE_TRANSACTIONAL = "TRANSACTIONAL"; + public static final String LAST_MOD_COLUMN = "LastModDate"; @Id @GeneratedValue(strategy = GenerationType.AUTO) @@ -70,8 +71,6 @@ public class DownSyncTableDetail { @Column(name = "TableType") private String tableType; - @Column(name = "LastModColumnName") - private String lastModColumnName; /** lower number syncs first - enforces the FK dependency chain */ @Expose @Column(name = "SyncOrder") @@ -139,12 +138,7 @@ public void setVanColumnName(String vanColumnName) { } public String getLastModColumnName() { - return (lastModColumnName == null || lastModColumnName.trim().isEmpty()) ? "LastModDate" - : lastModColumnName.trim(); - } - - public void setLastModColumnName(String lastModColumnName) { - this.lastModColumnName = lastModColumnName; + return LAST_MOD_COLUMN; } public String getVanAutoIncColumnName() { From 2c957c93ed47dc77506d1d6df303875ed85ae244 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 15:29:46 +0530 Subject: [PATCH 42/81] fix: update the response of downsync --- .../dataSyncActivity/StartSyncActivity.java | 8 + .../DownSyncTableResult.java | 159 ++++++++++++++++++ .../dataSyncActivity/DataSyncRepository.java | 23 ++- .../DownSyncDataFromServerImpl.java | 71 ++++++-- 4 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 28b5a063..e6d57a9d 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -286,6 +286,14 @@ public String startDownSync(@RequestBody String requestOBJ, try { String jwtToken = CookieUtil.getJwtTokenFromCookie(request); + if (serverAuthorization == null || serverAuthorization.trim().isEmpty()) { + logger.error("Down-sync : ServerAuthorization is empty - do the data sync server login first"); + response.setError(5000, + "Server authorization is missing. Kindly do the data sync server login and try again."); + return response.toString(); + } + logger.info("Down-sync : ServerAuthorization received, length {}", serverAuthorization.trim().length()); + JSONObject obj = new JSONObject(requestOBJ); if (obj.has("vanID") && obj.get("vanID") != null) { Integer providerServiceMapID = obj.has(PROVIDER_SERVICE_MAP_ID) && !obj.isNull(PROVIDER_SERVICE_MAP_ID) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java new file mode 100644 index 00000000..7c54b79d --- /dev/null +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java @@ -0,0 +1,159 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.data.syncActivity_syncLayer; + +import java.util.LinkedHashMap; +import java.util.Map; + +import com.google.gson.annotations.Expose; + +/*** + * The outcome of down-syncing one table, reported to the screen. + * + *

+ * A table can fail in two quite different ways and the difference matters when + * reading the result: + * + *

    + *
  • the table itself failed - central could not even return the rows (a schema + * mismatch, say), so nothing was delivered and no record was flagged;
    • + *
  • the table was delivered but individual records failed, in which case the + * table shows a record failure count and each distinct reason with a tally.
    • + *
+ */ +public class DownSyncTableResult { + + public static final String STATUS_SUCCESS = "SUCCESS"; + public static final String STATUS_FAILED = "FAILED"; + public static final String STATUS_PARTIAL = "PARTIAL"; + + @Expose + private String groupName; + @Expose + private String schemaName; + @Expose + private String tableName; + @Expose + private String status; + @Expose + private int inserted; + @Expose + private int updated; + @Expose + private int skipped; + @Expose + private int conflicts; + @Expose + private int failedRecords; + /** why the table as a whole failed; null when the table itself was fine */ + @Expose + private String failureReason; + /** distinct record-level failure reasons, each with the number of records */ + @Expose + private Map recordFailureReasons = new LinkedHashMap<>(); + + public DownSyncTableResult(String groupName, String schemaName, String tableName) { + this.groupName = groupName; + this.schemaName = schemaName; + this.tableName = tableName; + this.status = STATUS_SUCCESS; + } + + /** the table could not be processed at all - nothing was delivered */ + public void tableFailed(String reason) { + this.status = STATUS_FAILED; + this.failureReason = reason; + } + + /** one record failed; the table itself is fine unless every record failed */ + public void recordFailed(String reason) { + this.failedRecords++; + String key = (reason == null || reason.trim().isEmpty()) ? "Unknown error" : reason.trim(); + recordFailureReasons.merge(key, 1, Integer::sum); + if (STATUS_SUCCESS.equals(this.status)) + this.status = STATUS_PARTIAL; + } + + public int getRecordsProcessed() { + return inserted + updated + skipped + conflicts + failedRecords; + } + + public void addInserted() { + inserted++; + } + + public void addUpdated() { + updated++; + } + + public void addSkipped() { + skipped++; + } + + public void addConflict() { + conflicts++; + } + + public String getGroupName() { + return groupName; + } + + public String getSchemaName() { + return schemaName; + } + + public String getTableName() { + return tableName; + } + + public String getStatus() { + return status; + } + + public int getInserted() { + return inserted; + } + + public int getUpdated() { + return updated; + } + + public int getSkipped() { + return skipped; + } + + public int getConflicts() { + return conflicts; + } + + public int getFailedRecords() { + return failedRecords; + } + + public String getFailureReason() { + return failureReason; + } + + public Map getRecordFailureReasons() { + return recordFailureReasons; + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 6bbc33f8..ab068adb 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -23,6 +23,7 @@ import java.sql.Timestamp; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -143,13 +144,33 @@ public int[] updateLatestMasterInLocal(String query, List syncDataList // ---------------------------------- Down-Sync Repository (central -> local) + public Map getSyncGroupNamesByTable() { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT LOWER(TRIM(s.TableName)) AS TableName, g.SyncTableGroupName " + + " FROM db_iemr.m_synctabledetail s " + + " JOIN db_iemr.m_synctablegroup g ON g.SyncTableGroupID = s.SyncTableGroupID " + + " WHERE IFNULL(s.Deleted, b'0') = b'0' AND IFNULL(g.Deleted, b'0') = b'0' " + + " AND s.TableName IS NOT NULL "; + + Map groups = new LinkedHashMap<>(); + for (Map row : jdbcTemplate.queryForList(query)) { + Object table = row.get("TableName"); + Object group = row.get("SyncTableGroupName"); + if (table != null && group != null) + groups.putIfAbsent(String.valueOf(table), String.valueOf(group)); + } + return groups; + } + public List getDownSyncColumns(String schema, String table) { jdbcTemplate = getJdbcTemplate(); String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + " AND COLUMN_NAME NOT IN ('DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " - + " 'LastDownSyncDate') " + " ORDER BY COLUMN_NAME "; + + " 'LastDownSyncDate', 'Processed', 'SyncFailureReason', 'SyncedBy', 'SyncedDate') " + + " ORDER BY COLUMN_NAME "; return jdbcTemplate.queryForList(query, String.class, schema, table); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 632af829..870bcd9c 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -25,6 +25,7 @@ import java.text.DateFormat; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.Collections; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; @@ -51,6 +52,7 @@ import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncDataDigester; import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableDetail; +import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncTableResult; import com.iemr.mmu.repo.syncActivity_syncLayer.DownSyncTableDetailRepo; import com.iemr.mmu.utils.RestTemplateUtil; @@ -64,6 +66,7 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String VAN_SERIAL_NO = "VanSerialNo"; private static final String VAN_ID = "VanID"; private static final String LAST_MOD_DATE = "LastModDate"; + private static final String DATA_SYNC_CALL = "datasync"; private static final String LAST_DOWN_SYNC_DATE = "LastDownSyncDate"; private static final String DOWN_SYNCED = "DownSynced"; private static final String DOWN_SYNC_DATE = "DownSyncDate"; @@ -90,8 +93,12 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static int updatedCounter = 0; private static int conflictCounter = 0; private static int skippedCounter = 0; - private static int failedCounter = 0; + private static int failedTableCounter = 0; + private static int failedRecordCounter = 0; private static StringBuilder failedTables = new StringBuilder(); + private static final List tableResults = + Collections.synchronizedList(new ArrayList<>()); + private static DownSyncTableResult currentResult; private static String currentTable = ""; public String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, @@ -113,21 +120,35 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer updatedCounter = 0; conflictCounter = 0; skippedCounter = 0; - failedCounter = 0; + failedTableCounter = 0; + failedRecordCounter = 0; failedTables = new StringBuilder(); + tableResults.clear(); + currentResult = null; currentTable = ""; + final Map syncGroups = dataSyncRepository.getSyncGroupNamesByTable(); + final ExecutorService threadPool = Executors.newSingleThreadExecutor(); try { threadPool.submit(() -> { try { for (DownSyncTableDetail tableDetail : downSyncTables) { currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); + + String groupName = syncGroups.getOrDefault( + tableDetail.getTableName().toLowerCase(), + tableDetail.isMasterTable() ? "Masters" : "Other"); + currentResult = new DownSyncTableResult(groupName, tableDetail.getSchemaName(), + tableDetail.getTableName()); + tableResults.add(currentResult); + try { downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); } catch (Exception e) { - failedCounter++; + failedTableCounter++; failedTables.append(tableDetail.getTableName()).append(" | "); + currentResult.tableFailed(shorten(e.getMessage())); logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); } progressCounter++; @@ -151,8 +172,13 @@ public Map getDownSyncStatus() { resultMap.put("currentTable", currentTable); resultMap.put("totalTables", totalCounter); resultMap.put("completedTables", progressCounter); - resultMap.put("failedTableCount", failedCounter); + resultMap.put("successTableCount", progressCounter - failedTableCounter); + resultMap.put("failedTableCount", failedTableCounter); + resultMap.put("failedRecordCount", failedRecordCounter); resultMap.put("failedTables", failedTables.toString()); + synchronized (tableResults) { + resultMap.put("tableResults", new ArrayList<>(tableResults)); + } resultMap.put("recordsInserted", insertedCounter); resultMap.put("recordsUpdated", updatedCounter); resultMap.put("recordsSkipped", skippedCounter); @@ -215,7 +241,8 @@ private List> fetchDataFromCentral(DownSyncTableDetail table providerServiceMapID); RestTemplate restTemplate = new RestTemplate(); - HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, + DATA_SYNC_CALL); ResponseEntity response = restTemplate.exchange(downSyncDataUrl, HttpMethod.POST, request, String.class); if (response == null || !response.hasBody()) @@ -300,6 +327,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD if (localRecord == null) { Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); insertedCounter++; + if (currentResult != null) + currentResult.addInserted(); acks.add(DownSyncRecordAck.success(centralID, localID)); continue; } @@ -308,6 +337,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD if (!isCentralCopyNewer(record, localRecord, tableDetail.getLastModColumnName())) { skippedCounter++; + if (currentResult != null) + currentResult.addSkipped(); acks.add(DownSyncRecordAck.success(centralID, localID)); continue; } @@ -316,6 +347,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD dataSyncRepository.markDownSyncConflictInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, localID); conflictCounter++; + if (currentResult != null) + currentResult.addConflict(); acks.add(DownSyncRecordAck.failure(centralID, localID, DownSyncRecordAck.CONFLICT)); logger.warn("Down-sync conflict for {}.{}, local id {}", tableDetail.getSchemaName(), tableDetail.getTableName(), localID); @@ -324,13 +357,15 @@ private List saveTransactionalData(DownSyncTableDetail tableD updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, localID); updatedCounter++; + if (currentResult != null) + currentResult.addUpdated(); acks.add(DownSyncRecordAck.success(centralID, localID)); } catch (Exception e) { - failedCounter++; - String reason = e.getMessage() != null && e.getMessage().length() > 250 - ? e.getMessage().substring(0, 250) - : e.getMessage(); + failedRecordCounter++; + String reason = shorten(e.getMessage()); + if (currentResult != null) + currentResult.recordFailed(reason); acks.add(DownSyncRecordAck.failure(centralID, toLong(vanSerialNo), reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); @@ -381,6 +416,10 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo values.add(null); columns.add(LAST_DOWN_SYNC_DATE); values.add(Timestamp.valueOf(LocalDateTime.now())); + columns.add(DOWN_SYNCED); + values.add(DownSyncRecordAck.STATUS_PROCESSED); + columns.add(DOWN_SYNC_DATE); + values.add(Timestamp.valueOf(LocalDateTime.now())); StringBuilder placeHolders = new StringBuilder(); for (int i = 0; i < columns.size(); i++) { @@ -413,7 +452,9 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo } setClause.append(setClause.length() > 0 ? ", " : "").append(PROCESSED).append(" = 'P', ") - .append(SYNC_FAILURE_REASON).append(" = NULL, ").append(LAST_DOWN_SYNC_DATE).append(" = ? "); + .append(SYNC_FAILURE_REASON).append(" = NULL, ").append(LAST_DOWN_SYNC_DATE).append(" = ?, ") + .append(DOWN_SYNCED).append(" = 'P', ").append(DOWN_SYNC_DATE).append(" = ? "); + values.add(Timestamp.valueOf(LocalDateTime.now())); values.add(Timestamp.valueOf(LocalDateTime.now())); values.add(localID); @@ -423,6 +464,13 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); } + private static String shorten(String message) { + if (message == null || message.trim().isEmpty()) + return "Unknown error"; + String trimmed = message.trim(); + return trimmed.length() > 250 ? trimmed.substring(0, 250) : trimmed; + } + private void requireIdentifier(String value, String what, DownSyncTableDetail tableDetail) throws Exception { if (value == null || !value.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) throw new Exception("The down-sync configuration of " + tableDetail.getTableName() + " holds an invalid " @@ -444,7 +492,8 @@ private void acknowledgeToCentral(DownSyncTableDetail tableDetail, Integer vanID DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, acks); RestTemplate restTemplate = new RestTemplate(); - HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, jwtToken); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, + DATA_SYNC_CALL); ResponseEntity response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, String.class); From d0438c1989336ac35e9c3810484a75302e7abd14 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 15:40:44 +0530 Subject: [PATCH 43/81] fix: update the continuous call on check sync progress --- .../dataSyncActivity/StartSyncActivity.java | 7 +-- .../DownSyncDataFromServerImpl.java | 54 +++++++++---------- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index e6d57a9d..827aded6 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -303,10 +303,11 @@ public String startDownSync(@RequestBody String requestOBJ, String s = downSyncDataFromServerImpl.startDownSync(serverAuthorization, jwtToken, obj.getInt("vanID"), providerServiceMapID); - if (s != null && s.equalsIgnoreCase("inProgress")) + if (s != null && s.equalsIgnoreCase("inProgress")) { response.setError(5000, "Down-sync is already in progress, kindly wait for it to finish"); - else - response.setResponse(s); + } else { + response.setResponse(new Gson().toJson(downSyncDataFromServerImpl.getDownSyncStatus())); + } } else { response.setError(5000, "vanID is missing, Kindly contact the administrator."); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 870bcd9c..e0a19be0 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -30,8 +30,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import org.json.JSONObject; @@ -129,40 +127,36 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer final Map syncGroups = dataSyncRepository.getSyncGroupNamesByTable(); - final ExecutorService threadPool = Executors.newSingleThreadExecutor(); try { - threadPool.submit(() -> { + for (DownSyncTableDetail tableDetail : downSyncTables) { + currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); + + String groupName = syncGroups.getOrDefault(tableDetail.getTableName().toLowerCase(), + tableDetail.isMasterTable() ? "Masters" : "Other"); + currentResult = new DownSyncTableResult(groupName, tableDetail.getSchemaName(), + tableDetail.getTableName()); + tableResults.add(currentResult); + try { - for (DownSyncTableDetail tableDetail : downSyncTables) { - currentTable = tableDetail.getSchemaName() + "." + tableDetail.getTableName(); - - String groupName = syncGroups.getOrDefault( - tableDetail.getTableName().toLowerCase(), - tableDetail.isMasterTable() ? "Masters" : "Other"); - currentResult = new DownSyncTableResult(groupName, tableDetail.getSchemaName(), - tableDetail.getTableName()); - tableResults.add(currentResult); - - try { - downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); - } catch (Exception e) { - failedTableCounter++; - failedTables.append(tableDetail.getTableName()).append(" | "); - currentResult.tableFailed(shorten(e.getMessage())); - logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); - } - progressCounter++; - } - } finally { - IN_PROGRESS.set(false); + downSyncTable(tableDetail, vanID, providerServiceMapID, serverAuthorization, jwtToken); + } catch (Exception e) { + failedTableCounter++; + failedTables.append(tableDetail.getTableName()).append(" | "); + currentResult.tableFailed(shorten(e.getMessage())); + logger.error("Down-sync failed for " + currentTable + ". Exception : " + e.getMessage(), e); } - return "Down-sync completed"; - }); + progressCounter++; + } } finally { - threadPool.shutdown(); + IN_PROGRESS.set(false); + currentTable = ""; } - return " Down-sync started "; + logger.info("Down-sync finished : {} of {} tables succeeded, {} inserted, {} updated, {} skipped, " + + "{} conflicts, {} failed records", progressCounter - failedTableCounter, totalCounter, + insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter); + + return "Down-sync completed"; } public Map getDownSyncStatus() { From 6eb9221087344dcea97579055a3dedb836ab1221 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 16:22:43 +0530 Subject: [PATCH 44/81] fix: column name issue for stoptb tables --- .../service/dataSyncActivity/DataSyncRepository.java | 12 ++++++++++++ .../dataSyncActivity/DownSyncDataFromServerImpl.java | 12 +++++++++--- .../DataSyncRepositoryCentralDownload.java | 12 ++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index ab068adb..7f1d496d 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -163,6 +163,18 @@ public Map getSyncGroupNamesByTable() { return groups; } + public String resolveLastModColumn(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + + " AND COLUMN_NAME IN ('LastModDate', 'last_mod_date') " + + " ORDER BY FIELD(COLUMN_NAME, 'LastModDate', 'last_mod_date') "; + + List found = jdbcTemplate.queryForList(query, String.class, schema, table); + return (found == null || found.isEmpty()) ? null : found.get(0); + } + public List getDownSyncColumns(String schema, String table) { jdbcTemplate = getJdbcTemplate(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index e0a19be0..28bee1a8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -305,7 +305,13 @@ private List saveTransactionalData(DownSyncTableDetail tableD pkColumn = pkColumn.trim(); requireIdentifier(pkColumn, "VanAutoIncColumnName", tableDetail); - requireIdentifier(tableDetail.getLastModColumnName(), "LastModColumnName", tableDetail); + + String lastModColumn = dataSyncRepository.resolveLastModColumn(tableDetail.getSchemaName(), + tableDetail.getTableName()); + if (lastModColumn == null) + throw new Exception(tableDetail.getSchemaName() + "." + tableDetail.getTableName() + + " has neither LastModDate nor last_mod_date, so a change cannot be dated"); + requireIdentifier(lastModColumn, "modification-time column", tableDetail); List acks = new ArrayList<>(); @@ -316,7 +322,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD try { Map localRecord = dataSyncRepository.getLocalRecordForDownSync( tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, vanSerialNo, vanID, - tableDetail.getLastModColumnName()); + lastModColumn); if (localRecord == null) { Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); @@ -329,7 +335,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD Long localID = toLong(localRecord.get(pkColumn)); - if (!isCentralCopyNewer(record, localRecord, tableDetail.getLastModColumnName())) { + if (!isCentralCopyNewer(record, localRecord, lastModColumn)) { skippedCounter++; if (currentResult != null) currentResult.addSkipped(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 2cceb6d8..aa2db2a0 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -212,6 +212,18 @@ else if (masterType.equalsIgnoreCase("P")) { // ---------------------------------- Down-Sync Repository (central -> local) + public String resolveLastModColumn(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " + + " AND COLUMN_NAME IN ('LastModDate', 'last_mod_date') " + + " ORDER BY FIELD(COLUMN_NAME, 'LastModDate', 'last_mod_date') "; + + List found = jdbcTemplate.queryForList(query, String.class, schema, table); + return (found == null || found.isEmpty()) ? null : found.get(0); + } + public List> getDownSyncDataFromTable(String schema, String table, String columnNames, String tableType, Integer vanID, String lastModColumn) throws Exception { jdbcTemplate = getJdbcTemplate(); From af58fa01981e2933c888610bd4964cf215719fe2 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Thu, 27 Aug 2026 16:46:58 +0530 Subject: [PATCH 45/81] fix: issue in stoptb tables --- .../GetDownSyncDataFromCentralImpl.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java index 54b931ac..11d0bac2 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -63,24 +63,22 @@ public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) t List> resultSetList = dataSyncRepositoryCentralDownload.getDownSyncDataFromTable( downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), - downSyncDataDigester.getVanID(), resolveLastModColumn(downSyncDataDigester.getTableName())); + downSyncDataDigester.getVanID(), + resolveLastModColumn(downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName())); return downSyncGson().toJson(resultSetList); } - private String resolveLastModColumn(String tableName) throws Exception { - String lastModColumn = "LastModDate"; + private String resolveLastModColumn(String schemaName, String tableName) throws Exception { + String lastModColumn = dataSyncRepositoryCentralDownload.resolveLastModColumn(schemaName, tableName); - if (tableName != null) { - ArrayList tableDetails = downSyncTableDetailRepo - .getActiveDownSyncTableByName(tableName.trim()); - if (tableDetails != null && !tableDetails.isEmpty()) - lastModColumn = tableDetails.get(0).getLastModColumnName(); - } + if (lastModColumn == null) + throw new Exception(schemaName + "." + tableName + + " has neither LastModDate nor last_mod_date, so the down-sync cannot tell when a record was" + + " last changed"); - if (lastModColumn == null || !lastModColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) - throw new Exception( - "The down-sync configuration of " + tableName + " holds an invalid modification-time column name"); + if (!lastModColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + throw new Exception(schemaName + "." + tableName + " holds an invalid modification-time column name"); return lastModColumn; } From c4a41427bf5a1ea48d6e46aca39ef7312c59ad8d Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 11:19:14 +0530 Subject: [PATCH 46/81] fix: add central id --- .../dataSyncActivity/DataSyncRepository.java | 16 +++++++++---- .../DownSyncDataFromServerImpl.java | 23 +++++++++++++------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 7f1d496d..2ce57c7a 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -180,7 +180,7 @@ public List getDownSyncColumns(String schema, String table) { String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS " + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? " - + " AND COLUMN_NAME NOT IN ('DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " + + " AND COLUMN_NAME NOT IN ('CentralID', 'DownSynced', 'DownSyncDate', 'DownSyncFailureReason', " + " 'LastDownSyncDate', 'Processed', 'SyncFailureReason', 'SyncedBy', 'SyncedDate') " + " ORDER BY COLUMN_NAME "; @@ -188,16 +188,16 @@ public List getDownSyncColumns(String schema, String table) { } public Map getLocalRecordForDownSync(String schema, String table, String autoIncColumnName, - Object vanSerialNo, Object vanID, String lastModColumn) { - if (vanSerialNo == null) + Object centralID, Object vanID, String lastModColumn) { + if (centralID == null) return null; jdbcTemplate = getJdbcTemplate(); String query = " SELECT " + autoIncColumnName + ", Processed, " + lastModColumn + " AS LastModDate, LastDownSyncDate FROM " + schema + "." + table - + " WHERE " + autoIncColumnName + " = ? AND VanID = ? "; + + " WHERE CentralID = ? AND VanID = ? "; - List> resultSet = jdbcTemplate.queryForList(query, vanSerialNo, vanID); + List> resultSet = jdbcTemplate.queryForList(query, centralID, vanID); if (resultSet == null || resultSet.isEmpty()) return null; @@ -225,6 +225,12 @@ public int updateDownSyncRecordInLocal(String query, Object[] params) { return jdbcTemplate.update(query, params); } + public int updateVanSerialNoInLocal(String schema, String table, String autoIncColumnName, Object localID) { + jdbcTemplate = getJdbcTemplate(); + String query = " UPDATE " + schema + "." + table + " SET VanSerialNo = ? WHERE " + autoIncColumnName + " = ? "; + return jdbcTemplate.update(query, localID, localID); + } + public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); String query = " UPDATE " + schema + "." + table diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 28bee1a8..171051bd 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -71,6 +71,7 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String DOWN_SYNC_FAILURE_REASON = "DownSyncFailureReason"; private static final String PROCESSED = "Processed"; private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; + private static final String CENTRAL_ID = "CentralID"; @Value("${downSyncDataUrl}") private String downSyncDataUrl; @@ -317,15 +318,14 @@ private List saveTransactionalData(DownSyncTableDetail tableD for (Map record : dataFromCentral) { Long centralID = toLong(record.get(pkColumn)); - Object vanSerialNo = normalize(record.get(VAN_SERIAL_NO)); try { Map localRecord = dataSyncRepository.getLocalRecordForDownSync( - tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, vanSerialNo, vanID, + tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, centralID, vanID, lastModColumn); if (localRecord == null) { - Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record); + Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, centralID); insertedCounter++; if (currentResult != null) currentResult.addInserted(); @@ -366,7 +366,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD String reason = shorten(e.getMessage()); if (currentResult != null) currentResult.recordFailed(reason); - acks.add(DownSyncRecordAck.failure(centralID, toLong(vanSerialNo), reason)); + acks.add(DownSyncRecordAck.failure(centralID, null, reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); } @@ -395,7 +395,7 @@ private boolean hasUnsentLocalChanges(Map localRecord) { } private Long insertRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, - String pkColumn, Map record) { + String pkColumn, Map record, Long centralID) { List columns = new ArrayList<>(); List values = new ArrayList<>(); @@ -410,6 +410,8 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo values.add(normalize(record.get(serverColumns.get(i)))); } + columns.add(CENTRAL_ID); + values.add(centralID); columns.add(PROCESSED); values.add("P"); columns.add(SYNC_FAILURE_REASON); @@ -431,7 +433,13 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo String query = " INSERT INTO " + tableDetail.getSchemaName() + "." + tableDetail.getTableName() + " ( " + String.join(", ", columns) + " ) VALUES ( " + placeHolders + " ) "; - return dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + Long localID = dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + + if (localID != null) + dataSyncRepository.updateVanSerialNoInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), + pkColumn, localID); + + return localID; } private void updateRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, @@ -478,7 +486,8 @@ private void requireIdentifier(String value, String what, DownSyncTableDetail ta } private boolean isDownSyncManagedColumn(String column) { - return PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) + return VAN_SERIAL_NO.equalsIgnoreCase(column) || CENTRAL_ID.equalsIgnoreCase(column) + || PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) || LAST_DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNCED.equalsIgnoreCase(column) || DOWN_SYNC_DATE.equalsIgnoreCase(column) || DOWN_SYNC_FAILURE_REASON.equalsIgnoreCase(column); } From 0e957e31bdbb53c6c689ffb84b25a6eba215b4a0 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 12:43:41 +0530 Subject: [PATCH 47/81] fix: FK Mapping issue --- .../DownSyncTableDetail.java | 14 +- .../dataSyncActivity/DataSyncRepository.java | 27 +++ .../DownSyncDataFromServerImpl.java | 199 ++++++++++++++++-- 3 files changed, 221 insertions(+), 19 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index c7a0dd68..9ffadb84 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -70,8 +70,12 @@ public class DownSyncTableDetail { @Expose @Column(name = "TableType") private String tableType; + + @Expose + @Column(name = "FkColumnMapping") + private String fkColumnMapping; - /** lower number syncs first - enforces the FK dependency chain */ + @Expose @Column(name = "SyncOrder") private Integer syncOrder; @@ -149,6 +153,14 @@ public void setVanAutoIncColumnName(String vanAutoIncColumnName) { this.vanAutoIncColumnName = vanAutoIncColumnName; } + public String getFkColumnMapping() { + return fkColumnMapping; + } + + public void setFkColumnMapping(String fkColumnMapping) { + this.fkColumnMapping = fkColumnMapping; + } + public String getTableType() { return tableType; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 2ce57c7a..66ddfc03 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -231,6 +231,33 @@ public int updateVanSerialNoInLocal(String schema, String table, String autoIncC return jdbcTemplate.update(query, localID, localID); } + + public Long resolveLocalIdForCentralValue(String schema, String table, String pkColumn, Object centralValue, + Object vanID) { + if (centralValue == null) + return null; + + jdbcTemplate = getJdbcTemplate(); + String from = " FROM " + schema + "." + table + " WHERE "; + + Long resolved = queryForFirstLong(" SELECT " + pkColumn + from + " CentralID = ? AND VanID = ? ", centralValue, + vanID); + if (resolved != null) + return resolved; + + resolved = queryForFirstLong(" SELECT " + pkColumn + from + " VanSerialNo = ? AND VanID = ? ", centralValue, + vanID); + if (resolved != null) + return resolved; + + return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); + } + + private Long queryForFirstLong(String query, Object... params) { + List found = jdbcTemplate.queryForList(query, Long.class, params); + return (found == null || found.isEmpty()) ? null : found.get(0); + } + public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); String query = " UPDATE " + schema + "." + table diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 171051bd..b8482f1f 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -72,6 +72,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String PROCESSED = "Processed"; private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; private static final String CENTRAL_ID = "CentralID"; + + private static final int ACK_BATCH_SIZE = 500; @Value("${downSyncDataUrl}") private String downSyncDataUrl; @@ -314,18 +316,41 @@ private List saveTransactionalData(DownSyncTableDetail tableD + " has neither LastModDate nor last_mod_date, so a change cannot be dated"); requireIdentifier(lastModColumn, "modification-time column", tableDetail); + Map fkMapping = parseFkColumnMapping(tableDetail); + Map fkCache = new LinkedHashMap<>(); + List acks = new ArrayList<>(); - for (Map record : dataFromCentral) { - Long centralID = toLong(record.get(pkColumn)); + for (Map incoming : dataFromCentral) { + Map record = incoming; + // Resolved ignoring case : column names differ in case between tables + // (BenConsentID vs BenConsentId, vanSerialNo vs VanSerialNo) and a map + // lookup is case-sensitive, so taking the configured spelling literally + // yields null and every record then looks new on every run. + Long centralID = toLong(record.get(resolveKeyIgnoringCase(record, pkColumn))); try { + if (centralID == null) + throw new Exception("Central sent no value for the primary key '" + pkColumn + + "'. Check VanAutoIncColumnName in m_downsynctabledetail against the real column of " + + tableDetail.getTableName() + " - writing the record without it inserts a duplicate on" + + " every later run."); + Map localRecord = dataSyncRepository.getLocalRecordForDownSync( tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, centralID, vanID, lastModColumn); + // a pointer at another table's primary key means a different number here + // than at central, so it is rewritten before the row is written + StringBuilder unresolved = new StringBuilder(); + Map mapped = translateForeignKeys(fkMapping, record, vanID, fkCache, unresolved); + if (localRecord == null) { - Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, centralID); + if (mapped == null) + throw new Exception("Cannot place this record : " + shorten(unresolved.toString()) + + " Its parent row has not reached this van yet."); + + Long localID = insertRecord(tableDetail, serverColumns, vanColumns, pkColumn, mapped, centralID); insertedCounter++; if (currentResult != null) currentResult.addInserted(); @@ -355,7 +380,10 @@ private List saveTransactionalData(DownSyncTableDetail tableD continue; } - updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, record, localID); + // on an update an unresolvable pointer is left at its current local value + // rather than being overwritten with an id that means nothing here + updateRecord(tableDetail, serverColumns, vanColumns, pkColumn, mapped != null ? mapped : record, + localID, mapped == null ? fkMapping.keySet() : null); updatedCounter++; if (currentResult != null) currentResult.addUpdated(); @@ -443,7 +471,7 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo } private void updateRecord(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, - String pkColumn, Map record, Long localID) { + String pkColumn, Map record, Long localID, java.util.Set columnsToLeaveAlone) { StringBuilder setClause = new StringBuilder(); List values = new ArrayList<>(); @@ -452,6 +480,8 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo String vanColumn = vanColumns.get(i); if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) continue; + if (columnsToLeaveAlone != null && containsIgnoringCase(columnsToLeaveAlone, vanColumn)) + continue; if (setClause.length() > 0) setClause.append(", "); @@ -472,6 +502,121 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); } + /*** + * Parses FkColumnMapping - "childColumn:schema.parentTable", several separated + * by ';' - into an ordered map. Blank config gives an empty map, which is the + * "nothing to translate" case for almost every table. + */ + private Map parseFkColumnMapping(DownSyncTableDetail tableDetail) throws Exception { + Map mapping = new LinkedHashMap<>(); + String config = tableDetail.getFkColumnMapping(); + if (config == null || config.trim().isEmpty()) + return mapping; + + for (String entry : config.split(";")) { + if (entry == null || entry.trim().isEmpty()) + continue; + + String[] parts = entry.trim().split(":"); + if (parts.length != 2 || parts[1].trim().split("\\.").length != 2) + throw new Exception("FkColumnMapping of " + tableDetail.getTableName() + " holds '" + entry.trim() + + "', which is not of the form childColumn:schema.parentTable"); + + requireIdentifier(parts[0].trim(), "FkColumnMapping child column", tableDetail); + mapping.put(parts[0].trim(), parts[1].trim()); + } + return mapping; + } + + + private Map translateForeignKeys(Map fkMapping, Map record, + Integer vanID, Map cache, StringBuilder unresolved) { + + if (fkMapping.isEmpty()) + return record; + + Map translated = new LinkedHashMap<>(record); + + for (Map.Entry fk : fkMapping.entrySet()) { + String childColumn = resolveKeyIgnoringCase(record, fk.getKey()); + if (childColumn == null) + continue; + + Long centralValue = toLong(record.get(childColumn)); + if (centralValue == null) + continue; + + String parent = fk.getValue(); + String cacheKey = parent + "#" + centralValue; + Long localValue = cache.get(cacheKey); + + if (localValue == null) { + String parentSchema = parent.substring(0, parent.indexOf('.')); + String parentTable = parent.substring(parent.indexOf('.') + 1); + String parentPK = parentPrimaryKey(parentSchema, parentTable); + + if (parentPK == null) { + unresolved.append(fk.getKey()).append(" -> ").append(parent) + .append(" (no VanAutoIncColumnName configured for the parent); "); + continue; + } + + localValue = dataSyncRepository.resolveLocalIdForCentralValue(parentSchema, parentTable, parentPK, + centralValue, vanID); + if (localValue != null) + cache.put(cacheKey, localValue); + } + + if (localValue == null) { + unresolved.append(fk.getKey()).append('=').append(centralValue).append(" not found in ").append(parent) + .append("; "); + continue; + } + + translated.put(childColumn, localValue); + } + + return unresolved.length() > 0 ? null : translated; + } + + /*** + * The parent's local primary key, taken from its own down-sync configuration. + */ + private String parentPrimaryKey(String parentSchema, String parentTable) { + ArrayList parents = downSyncTableDetailRepo.getActiveDownSyncTableByName(parentTable); + for (DownSyncTableDetail parent : parents) { + if (parentSchema.equalsIgnoreCase(parent.getSchemaName()) && parent.getVanAutoIncColumnName() != null + && parent.getVanAutoIncColumnName().trim().matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + return parent.getVanAutoIncColumnName().trim(); + } + return null; + } + + /*** + * Column names differ in case between the tables (vanSerialNo vs VanSerialNo), + * and a map lookup is case-sensitive - so never index a result row by a + * hard-coded name without this. + */ + private String resolveKeyIgnoringCase(Map record, String column) { + if (record == null || column == null) + return null; + if (record.containsKey(column)) + return column; + for (String key : record.keySet()) { + if (key.equalsIgnoreCase(column)) + return key; + } + return null; + } + + private boolean containsIgnoringCase(java.util.Set columns, String column) { + for (String candidate : columns) { + if (candidate.equalsIgnoreCase(column)) + return true; + } + return false; + } + private static String shorten(String message) { if (message == null || message.trim().isEmpty()) return "Unknown error"; @@ -498,23 +643,41 @@ private void acknowledgeToCentral(DownSyncTableDetail tableDetail, Integer vanID if (acks == null || acks.isEmpty()) return; - DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, acks); - RestTemplate restTemplate = new RestTemplate(); - HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, - DATA_SYNC_CALL); - ResponseEntity response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, - String.class); + int acknowledged = 0; - if (response == null || !response.hasBody()) - throw new Exception("Empty response while updating the down-sync flag for " + tableDetail.getTableName()); + for (int from = 0; from < acks.size(); from += ACK_BATCH_SIZE) { + List batch = acks.subList(from, Math.min(from + ACK_BATCH_SIZE, acks.size())); - JSONObject responseObj = new JSONObject(response.getBody()); - if (!responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) - throw new Exception("Could not update the down-sync flag at central for " + tableDetail.getTableName() - + " : " + responseObj.optString("errorMessage")); + DownSyncDataDigester digester = DownSyncDataDigester.forAck(tableDetail, vanID, batch); + HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, + DATA_SYNC_CALL); + + ResponseEntity response; + try { + response = restTemplate.exchange(downSyncFlagUpdateUrl, HttpMethod.POST, request, String.class); + } catch (Exception e) { + // the batches already accepted stay acknowledged - say how far we got, so + // the gap between the two sides is visible rather than guessed at + throw new Exception("Down-sync flag update failed for " + tableDetail.getTableName() + " after " + + acknowledged + " of " + acks.size() + " records : " + e.getMessage(), e); + } + + if (response == null || !response.hasBody()) + throw new Exception("Empty response while updating the down-sync flag for " + + tableDetail.getTableName() + " after " + acknowledged + " of " + acks.size() + " records"); + + JSONObject responseObj = new JSONObject(response.getBody()); + if (!responseObj.has("statusCode") || responseObj.getInt("statusCode") != 200) + throw new Exception("Could not update the down-sync flag at central for " + + tableDetail.getTableName() + " after " + acknowledged + " of " + acks.size() + " records : " + + responseObj.optString("errorMessage")); + + acknowledged += batch.size(); + } - logger.info("Down-sync flag updated at central for {} records of {}", acks.size(), tableDetail.getTableName()); + logger.info("Down-sync flag updated at central for {} records of {} in {} batch(es)", acknowledged, + tableDetail.getTableName(), (acks.size() + ACK_BATCH_SIZE - 1) / ACK_BATCH_SIZE); } private List splitColumns(String columns) { From 48c98e859c064c31213843ae913d1a98f6d0c740 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 12:55:19 +0530 Subject: [PATCH 48/81] fix: downsync issue --- .../DownSyncDataDigester.java | 22 +++++- .../DownSyncDataFromServerImpl.java | 68 ++++++++++++++++--- .../DataSyncRepositoryCentralDownload.java | 29 ++++++-- .../GetDownSyncDataFromCentralImpl.java | 4 +- 4 files changed, 105 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java index af143161..c250cf73 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncDataDigester.java @@ -47,12 +47,14 @@ public class DownSyncDataDigester { /** populated only on the acknowledgement call */ @Expose private List records; + private Long lastFetchedID; + private Integer batchSize; public DownSyncDataDigester() { } public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, String serverColumnName, - Integer vanID, Integer providerServiceMapID) { + Integer vanID, Integer providerServiceMapID, Long lastFetchedID, Integer batchSize) { DownSyncDataDigester digester = new DownSyncDataDigester(); digester.schemaName = tableDetail.getSchemaName(); digester.tableName = tableDetail.getTableName(); @@ -61,6 +63,8 @@ public static DownSyncDataDigester forDownload(DownSyncTableDetail tableDetail, digester.tableType = tableDetail.getTableType(); digester.vanID = vanID; digester.providerServiceMapID = providerServiceMapID; + digester.lastFetchedID = lastFetchedID; + digester.batchSize = batchSize; return digester; } @@ -132,6 +136,22 @@ public void setProviderServiceMapID(Integer providerServiceMapID) { this.providerServiceMapID = providerServiceMapID; } + public Long getLastFetchedID() { + return lastFetchedID; + } + + public void setLastFetchedID(Long lastFetchedID) { + this.lastFetchedID = lastFetchedID; + } + + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + public List getRecords() { return records; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index b8482f1f..7e1396de 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -74,6 +74,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String CENTRAL_ID = "CentralID"; private static final int ACK_BATCH_SIZE = 500; + + private static final int FETCH_BATCH_SIZE = 1000; @Value("${downSyncDataUrl}") private String downSyncDataUrl; @@ -213,29 +215,75 @@ private void downSyncTable(DownSyncTableDetail tableDetail, Integer vanID, Integ + tableDetail.getTableName() + " (" + serverColumns.size() + " vs " + vanColumns.size() + "). Central and local must expose the same number of columns, in the same order."); - List> dataFromCentral = fetchDataFromCentral(tableDetail, String.join(",", serverColumns), - vanID, providerServiceMapID, serverAuthorization, jwtToken); + String columnList = String.join(",", serverColumns); - if (dataFromCentral.isEmpty()) { - logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), tableDetail.getTableName()); + if (tableDetail.isMasterTable()) { + // masters are a full pull with no primary key to page on, and are small + List> dataFromCentral = fetchDataFromCentral(tableDetail, columnList, vanID, + providerServiceMapID, serverAuthorization, jwtToken, null, null); + + if (dataFromCentral.isEmpty()) { + logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), + tableDetail.getTableName()); + return; + } + upsertMasterData(tableDetail, serverColumns, vanColumns, dataFromCentral); return; } - if (tableDetail.isMasterTable()) { - upsertMasterData(tableDetail, serverColumns, vanColumns, dataFromCentral); - } else { + String pkColumn = tableDetail.getVanAutoIncColumnName(); + requireIdentifier(pkColumn == null ? null : pkColumn.trim(), "VanAutoIncColumnName", tableDetail); + + Long lastFetchedID = null; + int page = 0; + int totalFetched = 0; + + while (true) { + List> dataFromCentral = fetchDataFromCentral(tableDetail, columnList, vanID, + providerServiceMapID, serverAuthorization, jwtToken, lastFetchedID, FETCH_BATCH_SIZE); + + if (dataFromCentral.isEmpty()) + break; + + page++; + totalFetched += dataFromCentral.size(); + List acks = saveTransactionalData(tableDetail, serverColumns, vanColumns, dataFromCentral, vanID); acknowledgeToCentral(tableDetail, vanID, acks, serverAuthorization, jwtToken); + + Long highestID = highestCentralID(dataFromCentral, pkColumn.trim()); + if (highestID == null || (lastFetchedID != null && highestID <= lastFetchedID)) + break; + lastFetchedID = highestID; + + if (dataFromCentral.size() < FETCH_BATCH_SIZE) + break; + } + + if (totalFetched == 0) + logger.info("Nothing to down-sync for {}.{}", tableDetail.getSchemaName(), tableDetail.getTableName()); + else + logger.info("Down-synced {} records of {}.{} in {} page(s)", totalFetched, tableDetail.getSchemaName(), + tableDetail.getTableName(), page); + } + + private Long highestCentralID(List> records, String pkColumn) { + Long highest = null; + for (Map record : records) { + Long id = toLong(record.get(resolveKeyIgnoringCase(record, pkColumn))); + if (id != null && (highest == null || id > highest)) + highest = id; } + return highest; } private List> fetchDataFromCentral(DownSyncTableDetail tableDetail, String serverColumnName, - Integer vanID, Integer providerServiceMapID, String serverAuthorization, String jwtToken) - throws Exception { + Integer vanID, Integer providerServiceMapID, String serverAuthorization, String jwtToken, + Long lastFetchedID, Integer batchSize) throws Exception { DownSyncDataDigester digester = DownSyncDataDigester.forDownload(tableDetail, serverColumnName, vanID, - providerServiceMapID); + providerServiceMapID, lastFetchedID, batchSize); RestTemplate restTemplate = new RestTemplate(); HttpEntity request = RestTemplateUtil.createRequestEntity(digester, serverAuthorization, diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index aa2db2a0..f9c2c557 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -225,7 +225,8 @@ public String resolveLastModColumn(String schema, String table) { } public List> getDownSyncDataFromTable(String schema, String table, String columnNames, - String tableType, Integer vanID, String lastModColumn) throws Exception { + String tableType, Integer vanID, String lastModColumn, String pkColumn, Long lastFetchedID, + Integer batchSize) throws Exception { jdbcTemplate = getJdbcTemplate(); if (schema == null || table == null) @@ -238,12 +239,28 @@ public List> getDownSyncDataFromTable(String schema, String if (vanID == null) throw new Exception("Invalid down-sync request. VanID is mandatory for transactional table " + table); - String query = " SELECT " + columns + " FROM " + schema + "." + table - + " WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') " - + " OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND " + lastModColumn - + " > DownSyncDate ) ) "; + boolean paged = pkColumn != null && pkColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$") && batchSize != null + && batchSize > 0; + + StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(schema) + .append(".").append(table) + .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') ") + .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(lastModColumn) + .append(" > DownSyncDate ) ) "); + + List params = new ArrayList<>(); + params.add(vanID); + + if (paged) { + if (lastFetchedID != null) { + query.append(" AND ").append(pkColumn).append(" > ? "); + params.add(lastFetchedID); + } + query.append(" ORDER BY ").append(pkColumn).append(" LIMIT ").append(batchSize.intValue()); + } + logger.info("Down-sync select query for {}.{} : {}", schema, table, query); - resultSetList = jdbcTemplate.queryForList(query, vanID); + resultSetList = jdbcTemplate.queryForList(query.toString(), params.toArray()); } else { String query = " SELECT " + columns + " FROM " + schema + "." + table; logger.info("Down-sync select query for {}.{} : {}", schema, table, query); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java index 11d0bac2..8dc1015f 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDownSyncDataFromCentralImpl.java @@ -64,7 +64,9 @@ public String getDownSyncDataForVan(DownSyncDataDigester downSyncDataDigester) t downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName(), downSyncDataDigester.getServerColumnName(), downSyncDataDigester.getTableType(), downSyncDataDigester.getVanID(), - resolveLastModColumn(downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName())); + resolveLastModColumn(downSyncDataDigester.getSchemaName(), downSyncDataDigester.getTableName()), + downSyncDataDigester.getVanAutoIncColumnName(), downSyncDataDigester.getLastFetchedID(), + downSyncDataDigester.getBatchSize()); return downSyncGson().toJson(resultSetList); } From 4c11d31502346f1bece4e0e12cd62d0fa3f76eca Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 13:24:55 +0530 Subject: [PATCH 49/81] fix: FK issue in i_ beneficiarymapping table --- .../DownSyncTableResult.java | 9 ++++++++ .../dataSyncActivity/DataSyncRepository.java | 9 ++++++++ .../DownSyncDataFromServerImpl.java | 22 ++++++++++++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java index 7c54b79d..8d253f05 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java @@ -70,6 +70,7 @@ public class DownSyncTableResult { /** distinct record-level failure reasons, each with the number of records */ @Expose private Map recordFailureReasons = new LinkedHashMap<>(); + private Map recordWarnings = new LinkedHashMap<>(); public DownSyncTableResult(String groupName, String schemaName, String tableName) { this.groupName = groupName; @@ -93,6 +94,14 @@ public void recordFailed(String reason) { this.status = STATUS_PARTIAL; } + public void recordWarning(String reason) { + recordWarnings.merge(reason, 1, Integer::sum); + } + + public Map getRecordWarnings() { + return recordWarnings; + } + public int getRecordsProcessed() { return inserted + updated + skipped + conflicts + failedRecords; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 66ddfc03..524ab899 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -253,6 +253,15 @@ public Long resolveLocalIdForCentralValue(String schema, String table, String pk return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); } + public boolean isColumnNullable(String schema, String table, String column) { + jdbcTemplate = getJdbcTemplate(); + List nullable = jdbcTemplate.queryForList( + " SELECT IS_NULLABLE FROM information_schema.COLUMNS " + + " WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND COLUMN_NAME = ? ", + String.class, schema, table, column); + return !nullable.isEmpty() && "YES".equalsIgnoreCase(nullable.get(0)); + } + private Long queryForFirstLong(String query, Object... params) { List found = jdbcTemplate.queryForList(query, Long.class, params); return (found == null || found.isEmpty()) ? null : found.get(0); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 7e1396de..81b537a5 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -391,7 +391,8 @@ private List saveTransactionalData(DownSyncTableDetail tableD // a pointer at another table's primary key means a different number here // than at central, so it is rewritten before the row is written StringBuilder unresolved = new StringBuilder(); - Map mapped = translateForeignKeys(fkMapping, record, vanID, fkCache, unresolved); + Map mapped = translateForeignKeys(tableDetail, fkMapping, record, vanID, fkCache, + unresolved); if (localRecord == null) { if (mapped == null) @@ -577,8 +578,8 @@ private Map parseFkColumnMapping(DownSyncTableDetail tableDetail } - private Map translateForeignKeys(Map fkMapping, Map record, - Integer vanID, Map cache, StringBuilder unresolved) { + private Map translateForeignKeys(DownSyncTableDetail tableDetail, Map fkMapping, + Map record, Integer vanID, Map cache, StringBuilder unresolved) { if (fkMapping.isEmpty()) return record; @@ -616,6 +617,21 @@ private Map translateForeignKeys(Map fkMapping, } if (localValue == null) { + // A pointer the van cannot place is only fatal if the column demands a + // value. Where it is optional - a consent or an image the van has no copy + // of - the record is worth more than the pointer, so it lands with the + // column empty and the loss is reported rather than hidden. + if (dataSyncRepository.isColumnNullable(tableDetail.getSchemaName(), tableDetail.getTableName(), + childColumn)) { + translated.put(childColumn, null); + String warning = fk.getKey() + " could not be placed (" + parent + + " has no such row on this van) and was left empty"; + if (currentResult != null) + currentResult.recordWarning(warning); + logger.warn("{}.{} : {}", tableDetail.getSchemaName(), tableDetail.getTableName(), warning); + continue; + } + unresolved.append(fk.getKey()).append('=').append(centralValue).append(" not found in ").append(parent) .append("; "); continue; From e267058bb39891e6fa5afd1a5184edf83f9e9315 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 14:04:46 +0530 Subject: [PATCH 50/81] fix: include U for downsync again --- .../data/syncActivity_syncLayer/DownSyncRecordAck.java | 10 ++++++++++ .../dataSyncActivity/DownSyncDataFromServerImpl.java | 2 +- .../DataSyncRepositoryCentralDownload.java | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java index 5be35f81..e0de23c2 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncRecordAck.java @@ -27,6 +27,7 @@ public class DownSyncRecordAck { public static final String STATUS_PROCESSED = "P"; public static final String STATUS_FAILED = "F"; + public static final String STATUS_RETRY = "U"; public static final String CONFLICT = "CONFLICT"; @Expose @@ -56,6 +57,15 @@ public static DownSyncRecordAck failure(Long centralID, Long vanSerialNo, String return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_FAILED, failureReason); } + + public static DownSyncRecordAck retryable(Long centralID, Long vanSerialNo, String failureReason) { + return new DownSyncRecordAck(centralID, vanSerialNo, STATUS_RETRY, failureReason); + } + + public boolean isRetryable() { + return STATUS_RETRY.equalsIgnoreCase(status); + } + public boolean isSuccess() { return STATUS_PROCESSED.equalsIgnoreCase(status); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 81b537a5..0fd0c509 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -443,7 +443,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD String reason = shorten(e.getMessage()); if (currentResult != null) currentResult.recordFailed(reason); - acks.add(DownSyncRecordAck.failure(centralID, null, reason)); + acks.add(DownSyncRecordAck.retryable(centralID, null, reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index f9c2c557..38005752 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -283,6 +283,7 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk List successWithSerialNo = new ArrayList<>(); List successWithoutSerialNo = new ArrayList<>(); List failed = new ArrayList<>(); + List retryable = new ArrayList<>(); for (DownSyncRecordAck record : records) { if (record == null || record.getCentralID() == null) @@ -293,6 +294,8 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk successWithSerialNo.add(new Object[] { record.getVanSerialNo(), record.getCentralID() }); else successWithoutSerialNo.add(new Object[] { record.getCentralID() }); + } else if (record.isRetryable()) { + retryable.add(new Object[] { record.getFailureReason(), record.getCentralID() }); } else { failed.add(new Object[] { record.getFailureReason(), record.getCentralID() }); } @@ -314,6 +317,12 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); } + if (!retryable.isEmpty()) { + String query = " UPDATE " + schema + "." + table + + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, retryable)); + } + if (!failed.isEmpty()) { String query = " UPDATE " + schema + "." + table + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; From 7dc70ef56d08f2dfc6573b0fa38b7402a4b7f9fd Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 14:51:40 +0530 Subject: [PATCH 51/81] fix: revert central id for m_beneficiaryregidmapping --- .../syncActivity_syncLayer/DownSyncTableDetail.java | 12 ++++++++++++ .../dataSyncActivity/DownSyncDataFromServerImpl.java | 11 +++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java index 9ffadb84..1e22fdb6 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableDetail.java @@ -74,6 +74,10 @@ public class DownSyncTableDetail { @Expose @Column(name = "FkColumnMapping") private String fkColumnMapping; + + @Expose + @Column(name = "PreserveCentralPK") + private Boolean preserveCentralPK; @Expose @@ -153,6 +157,14 @@ public void setVanAutoIncColumnName(String vanAutoIncColumnName) { this.vanAutoIncColumnName = vanAutoIncColumnName; } + public boolean isPreserveCentralPK() { + return Boolean.TRUE.equals(preserveCentralPK); + } + + public void setPreserveCentralPK(Boolean preserveCentralPK) { + this.preserveCentralPK = preserveCentralPK; + } + public String getFkColumnMapping() { return fkColumnMapping; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 0fd0c509..87aed76e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -477,10 +477,11 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo List columns = new ArrayList<>(); List values = new ArrayList<>(); + boolean preservePK = tableDetail.isPreserveCentralPK(); + for (int i = 0; i < vanColumns.size(); i++) { String vanColumn = vanColumns.get(i); - // the local auto-increment PK is skipped so that local generates its own value - if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + if ((vanColumn.equalsIgnoreCase(pkColumn) && !preservePK) || isDownSyncManagedColumn(vanColumn)) continue; columns.add(vanColumn); @@ -512,6 +513,12 @@ private Long insertRecord(DownSyncTableDetail tableDetail, List serverCo Long localID = dataSyncRepository.insertDownSyncRecordInLocal(query, values.toArray()); + if (preservePK && localID == null) + localID = centralID; + + if (localID == null && preservePK) + localID = centralID; + if (localID != null) dataSyncRepository.updateVanSerialNoInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, localID); From 54425266f80f4e494ee842e04ea65f80f1587b3d Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 15:20:46 +0530 Subject: [PATCH 52/81] fix: conflict section --- .../DownSyncTableResult.java | 3 +++ .../dataSyncActivity/DataSyncRepository.java | 3 ++- .../DownSyncDataFromServerImpl.java | 23 ++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java index 8d253f05..a9691fbb 100644 --- a/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java +++ b/src/main/java/com/iemr/mmu/data/syncActivity_syncLayer/DownSyncTableResult.java @@ -45,6 +45,7 @@ public class DownSyncTableResult { public static final String STATUS_SUCCESS = "SUCCESS"; public static final String STATUS_FAILED = "FAILED"; public static final String STATUS_PARTIAL = "PARTIAL"; + public static final String STATUS_CONFLICT = "CONFLICT"; @Expose private String groupName; @@ -120,6 +121,8 @@ public void addSkipped() { public void addConflict() { conflicts++; + if (STATUS_SUCCESS.equals(this.status)) + this.status = STATUS_CONFLICT; } public String getGroupName() { diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 524ab899..97e7fcd5 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -270,7 +270,8 @@ private Long queryForFirstLong(String query, Object... params) { public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); String query = " UPDATE " + schema + "." + table - + " SET Processed = 'F', SyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; + + " SET Processed = 'F', SyncFailureReason = 'CONFLICT'," + + " DownSynced = 'F', DownSyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; return jdbcTemplate.update(query, localID); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 87aed76e..963d4526 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -161,7 +161,8 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer + "{} conflicts, {} failed records", progressCounter - failedTableCounter, totalCounter, insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter); - return "Down-sync completed"; + return conflictCounter > 0 ? "Down-sync completed with " + conflictCounter + " conflict(s) to review" + : "Down-sync completed"; } public Map getDownSyncStatus() { @@ -182,9 +183,29 @@ public Map getDownSyncStatus() { resultMap.put("recordsUpdated", updatedCounter); resultMap.put("recordsSkipped", skippedCounter); resultMap.put("conflicts", conflictCounter); + resultMap.put("conflictsPending", conflictCounter > 0); + resultMap.put("summary", buildSummary()); return resultMap; } + private String buildSummary() { + StringBuilder summary = new StringBuilder(); + summary.append(progressCounter - failedTableCounter).append(" of ").append(totalCounter) + .append(" tables synced, ").append(insertedCounter).append(" inserted, ").append(updatedCounter) + .append(" updated, ").append(skippedCounter).append(" unchanged"); + + if (failedTableCounter > 0) + summary.append(". ").append(failedTableCounter).append(" table(s) failed"); + if (failedRecordCounter > 0) + summary.append(". ").append(failedRecordCounter).append(" record(s) failed"); + if (conflictCounter > 0) + summary.append(". ").append(conflictCounter) + .append(conflictCounter == 1 ? " record is in conflict" : " records are in conflict") + .append(" and needs review before it can sync"); + + return summary.toString(); + } + /*** * Down-syncs one configured table. */ From 1b65b93cbdf348d4ee9aa6507aba4acaa90d2931 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 1 Sep 2026 15:31:19 +0530 Subject: [PATCH 53/81] fix: manage conflicts and add F flag for resync --- .../dataSyncActivity/DataSyncRepository.java | 8 +++++ .../DownSyncDataFromServerImpl.java | 35 +++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 97e7fcd5..9a222303 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -253,6 +253,14 @@ public Long resolveLocalIdForCentralValue(String schema, String table, String pk return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); } + public int countConflictsInLocal(String schema, String table, Object vanID) { + jdbcTemplate = getJdbcTemplate(); + String query = " SELECT COUNT(*) FROM " + schema + "." + table + + " WHERE VanID = ? AND Processed = 'F' AND SyncFailureReason = 'CONFLICT' "; + Integer count = jdbcTemplate.queryForObject(query, Integer.class, vanID); + return count == null ? 0 : count; + } + public boolean isColumnNullable(String schema, String table, String column) { jdbcTemplate = getJdbcTemplate(); List nullable = jdbcTemplate.queryForList( diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 963d4526..dfa92f42 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -103,6 +103,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { Collections.synchronizedList(new ArrayList<>()); private static DownSyncTableResult currentResult; private static String currentTable = ""; + private static int outstandingConflictCounter = 0; + private static final Map outstandingConflictTables = new LinkedHashMap<>(); public String startDownSync(String serverAuthorization, String jwtToken, Integer vanID, Integer providerServiceMapID) throws Exception { @@ -152,6 +154,7 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer } progressCounter++; } + countOutstandingConflicts(downSyncTables, vanID); } finally { IN_PROGRESS.set(false); currentTable = ""; @@ -183,11 +186,36 @@ public Map getDownSyncStatus() { resultMap.put("recordsUpdated", updatedCounter); resultMap.put("recordsSkipped", skippedCounter); resultMap.put("conflicts", conflictCounter); - resultMap.put("conflictsPending", conflictCounter > 0); + resultMap.put("outstandingConflicts", outstandingConflictCounter); + synchronized (outstandingConflictTables) { + resultMap.put("outstandingConflictTables", new LinkedHashMap<>(outstandingConflictTables)); + } + resultMap.put("conflictsPending", conflictCounter > 0 || outstandingConflictCounter > 0); resultMap.put("summary", buildSummary()); return resultMap; } + private void countOutstandingConflicts(List downSyncTables, Integer vanID) { + outstandingConflictCounter = 0; + outstandingConflictTables.clear(); + + for (DownSyncTableDetail tableDetail : downSyncTables) { + if (!tableDetail.isTransactionalTable()) + continue; + try { + int conflicts = dataSyncRepository.countConflictsInLocal(tableDetail.getSchemaName(), + tableDetail.getTableName(), vanID); + if (conflicts > 0) { + outstandingConflictCounter += conflicts; + outstandingConflictTables.put(tableDetail.getTableName(), conflicts); + } + } catch (Exception e) { + logger.warn("Could not count outstanding conflicts for {}.{} : {}", tableDetail.getSchemaName(), + tableDetail.getTableName(), e.getMessage()); + } + } + } + private String buildSummary() { StringBuilder summary = new StringBuilder(); summary.append(progressCounter - failedTableCounter).append(" of ").append(totalCounter) @@ -200,7 +228,10 @@ private String buildSummary() { summary.append(". ").append(failedRecordCounter).append(" record(s) failed"); if (conflictCounter > 0) summary.append(". ").append(conflictCounter) - .append(conflictCounter == 1 ? " record is in conflict" : " records are in conflict") + .append(conflictCounter == 1 ? " new conflict" : " new conflicts"); + if (outstandingConflictCounter > 0) + summary.append(". ").append(outstandingConflictCounter) + .append(outstandingConflictCounter == 1 ? " record is in conflict" : " records are in conflict") .append(" and needs review before it can sync"); return summary.toString(); From a4c35a9359aaaa28fdd1f9eefc4c6edfc9d0d20f Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 11:13:32 +0530 Subject: [PATCH 54/81] fix: pom version update --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 27d4ffdd..6157ff32 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.iemr.mmu mmu-api - 3.6.2 + 3.8.4 war MMU-API From ae5a424681740eebd644ebcda3d33fa9e37f934f Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 13:00:59 +0530 Subject: [PATCH 55/81] fix: add the condition to block the download from different van --- .../DownSyncDataFromServerImpl.java | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index dfa92f42..e65d05fa 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -94,6 +94,8 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static int progressCounter = 0; private static int insertedCounter = 0; private static int updatedCounter = 0; + private static int transactionalFetchedCounter = 0; + private static int masterUpdatedCounter = 0; private static int conflictCounter = 0; private static int skippedCounter = 0; private static int failedTableCounter = 0; @@ -123,6 +125,8 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer progressCounter = 0; insertedCounter = 0; updatedCounter = 0; + transactionalFetchedCounter = 0; + masterUpdatedCounter = 0; conflictCounter = 0; skippedCounter = 0; failedTableCounter = 0; @@ -160,12 +164,13 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer currentTable = ""; } - logger.info("Down-sync finished : {} of {} tables succeeded, {} inserted, {} updated, {} skipped, " - + "{} conflicts, {} failed records", progressCounter - failedTableCounter, totalCounter, - insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter); + logger.info("Down-sync finished : {} of {} tables succeeded, {} records delivered for van {}, {} inserted," + + " {} updated, {} skipped, {} conflicts, {} failed records, {} master rows refreshed", + progressCounter - failedTableCounter, totalCounter, transactionalFetchedCounter, vanID, + insertedCounter, updatedCounter, skippedCounter, conflictCounter, failedRecordCounter, + masterUpdatedCounter); - return conflictCounter > 0 ? "Down-sync completed with " + conflictCounter + " conflict(s) to review" - : "Down-sync completed"; + return buildSummary(); } public Map getDownSyncStatus() { @@ -191,7 +196,9 @@ public Map getDownSyncStatus() { resultMap.put("outstandingConflictTables", new LinkedHashMap<>(outstandingConflictTables)); } resultMap.put("conflictsPending", conflictCounter > 0 || outstandingConflictCounter > 0); + resultMap.put("recordsFoundForVan", transactionalFetchedCounter); resultMap.put("summary", buildSummary()); + resultMap.put("message", buildSummary()); return resultMap; } @@ -218,6 +225,17 @@ private void countOutstandingConflicts(List downSyncTables, private String buildSummary() { StringBuilder summary = new StringBuilder(); + + if (transactionalFetchedCounter == 0 && failedTableCounter == 0) { + summary.append("There is no record to down-sync for your van"); + if (outstandingConflictCounter > 0) + summary.append(". ").append(outstandingConflictCounter) + .append(outstandingConflictCounter == 1 ? " record is in conflict" + : " records are in conflict") + .append(" and needs review before it can sync"); + return summary.toString(); + } + summary.append(progressCounter - failedTableCounter).append(" of ").append(totalCounter) .append(" tables synced, ").append(insertedCounter).append(" inserted, ").append(updatedCounter) .append(" updated, ").append(skippedCounter).append(" unchanged"); @@ -299,6 +317,7 @@ private void downSyncTable(DownSyncTableDetail tableDetail, Integer vanID, Integ page++; totalFetched += dataFromCentral.size(); + transactionalFetchedCounter += dataFromCentral.size(); List acks = saveTransactionalData(tableDetail, serverColumns, vanColumns, dataFromCentral, vanID); @@ -389,7 +408,7 @@ private void upsertMasterData(DownSyncTableDetail tableDetail, List serv } int[] result = dataSyncRepository.updateLatestMasterInLocal(query, batch); - updatedCounter += result != null ? result.length : 0; + masterUpdatedCounter += result != null ? result.length : 0; logger.info("Down-synced {} master records into {}.{}", batch.size(), tableDetail.getSchemaName(), tableDetail.getTableName()); } From 2fedd6d814c297bfe23963a8acdbf47eb5792f56 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 14:17:13 +0530 Subject: [PATCH 56/81] fix: resolve foreign key violation --- .../dataSyncActivity/DataSyncRepository.java | 25 +++++++ .../DownSyncDataFromServerImpl.java | 72 +++++++++++++++---- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 9a222303..73eb47a8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -261,6 +261,31 @@ public int countConflictsInLocal(String schema, String table, Object vanID) { return count == null ? 0 : count; } + public List> getForeignKeysOfTable(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + String query = " SELECT k.CONSTRAINT_NAME, k.COLUMN_NAME AS CHILD_COLUMN," + + " k.REFERENCED_TABLE_SCHEMA AS PARENT_SCHEMA, k.REFERENCED_TABLE_NAME AS PARENT_TABLE," + + " k.REFERENCED_COLUMN_NAME AS PARENT_COLUMN " + + " FROM information_schema.KEY_COLUMN_USAGE k " + + " WHERE k.TABLE_SCHEMA = ? AND k.TABLE_NAME = ? AND k.REFERENCED_TABLE_NAME IS NOT NULL "; + + List> foreignKeys = jdbcTemplate.queryForList(query, schema, table); + if (foreignKeys == null) + return new ArrayList<>(); + + Map columnsPerConstraint = new LinkedHashMap<>(); + for (Map foreignKey : foreignKeys) { + columnsPerConstraint.merge(String.valueOf(foreignKey.get("CONSTRAINT_NAME")), 1, Integer::sum); + } + + List> singleColumn = new ArrayList<>(); + for (Map foreignKey : foreignKeys) { + if (columnsPerConstraint.get(String.valueOf(foreignKey.get("CONSTRAINT_NAME"))) == 1) + singleColumn.add(foreignKey); + } + return singleColumn; + } + public boolean isColumnNullable(String schema, String table, String column) { jdbcTemplate = getJdbcTemplate(); List nullable = jdbcTemplate.queryForList( diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index e65d05fa..849be15d 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -413,10 +413,7 @@ private void upsertMasterData(DownSyncTableDetail tableDetail, List serv tableDetail.getTableName()); } - /*** - * TRANSACTIONAL tables : record by record, so that a conflict on one record - * does not block the delivery of the rest. - */ + private List saveTransactionalData(DownSyncTableDetail tableDetail, List serverColumns, List vanColumns, List> dataFromCentral, Integer vanID) throws Exception { @@ -435,17 +432,13 @@ private List saveTransactionalData(DownSyncTableDetail tableD + " has neither LastModDate nor last_mod_date, so a change cannot be dated"); requireIdentifier(lastModColumn, "modification-time column", tableDetail); - Map fkMapping = parseFkColumnMapping(tableDetail); + Map fkMapping = resolveFkColumnMapping(tableDetail); Map fkCache = new LinkedHashMap<>(); List acks = new ArrayList<>(); for (Map incoming : dataFromCentral) { Map record = incoming; - // Resolved ignoring case : column names differ in case between tables - // (BenConsentID vs BenConsentId, vanSerialNo vs VanSerialNo) and a map - // lookup is case-sensitive, so taking the configured spelling literally - // yields null and every record then looks new on every run. Long centralID = toLong(record.get(resolveKeyIgnoringCase(record, pkColumn))); try { @@ -629,11 +622,62 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo dataSyncRepository.updateDownSyncRecordInLocal(query, values.toArray()); } - /*** - * Parses FkColumnMapping - "childColumn:schema.parentTable", several separated - * by ';' - into an ordered map. Blank config gives an empty map, which is the - * "nothing to translate" case for almost every table. - */ + private Map resolveFkColumnMapping(DownSyncTableDetail tableDetail) throws Exception { + Map configured = parseFkColumnMapping(tableDetail); + if (!configured.isEmpty()) + return configured; + + Map mapping = new LinkedHashMap<>(); + String ownPK = tableDetail.getVanAutoIncColumnName() == null ? null + : tableDetail.getVanAutoIncColumnName().trim(); + + for (Map foreignKey : dataSyncRepository.getForeignKeysOfTable(tableDetail.getSchemaName(), + tableDetail.getTableName())) { + + String childColumn = asTrimmedString(foreignKey.get("CHILD_COLUMN")); + String parentSchema = asTrimmedString(foreignKey.get("PARENT_SCHEMA")); + String parentTable = asTrimmedString(foreignKey.get("PARENT_TABLE")); + String parentColumn = asTrimmedString(foreignKey.get("PARENT_COLUMN")); + + if (childColumn == null || parentSchema == null || parentTable == null || parentColumn == null) + continue; + if (childColumn.equalsIgnoreCase(ownPK)) + continue; + + String parentPK = downSyncedTransactionalPrimaryKey(parentSchema, parentTable); + if (parentPK == null || !parentPK.equalsIgnoreCase(parentColumn)) + continue; + + requireIdentifier(childColumn, "foreign key column", tableDetail); + mapping.put(childColumn, parentSchema + "." + parentTable); + } + + if (!mapping.isEmpty()) + logger.info("Down-sync will rewrite {} of {}.{} from the table\'s own foreign keys", mapping, + tableDetail.getSchemaName(), tableDetail.getTableName()); + + return mapping; + } + + private String downSyncedTransactionalPrimaryKey(String parentSchema, String parentTable) { + ArrayList parents = downSyncTableDetailRepo.getActiveDownSyncTableByName(parentTable); + for (DownSyncTableDetail parent : parents) { + if (!parentSchema.equalsIgnoreCase(parent.getSchemaName()) || !parent.isTransactionalTable()) + continue; + if (parent.getVanAutoIncColumnName() != null + && parent.getVanAutoIncColumnName().trim().matches("^[a-zA-Z_][a-zA-Z0-9_]*$")) + return parent.getVanAutoIncColumnName().trim(); + } + return null; + } + + private String asTrimmedString(Object value) { + if (value == null) + return null; + String trimmed = String.valueOf(value).trim(); + return trimmed.isEmpty() ? null : trimmed; + } + private Map parseFkColumnMapping(DownSyncTableDetail tableDetail) throws Exception { Map mapping = new LinkedHashMap<>(); String config = tableDetail.getFkColumnMapping(); From ddf96cda33cdb79445d276f8d7558ab38521601c Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 14:35:39 +0530 Subject: [PATCH 57/81] fix: retry failed records --- .../DataSyncRepositoryCentralDownload.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 38005752..ba4692eb 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -244,9 +244,11 @@ public List> getDownSyncDataFromTable(String schema, String StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(schema) .append(".").append(table) - .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U') ") + .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U', 'F') ") .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(lastModColumn) - .append(" > DownSyncDate ) ) "); + .append(" > DownSyncDate ) ) ") + .append(" AND ( DownSyncFailureReason IS NULL OR DownSyncFailureReason <> '") + .append(DownSyncRecordAck.CONFLICT).append("' ) "); List params = new ArrayList<>(); params.add(vanID); From f1c5078ec8b77c34f88ef2a44376e70d4b5721eb Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 2 Sep 2026 15:11:37 +0530 Subject: [PATCH 58/81] fix: update 'F' flag --- .../service/dataSyncActivity/DownSyncDataFromServerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 849be15d..795ff80b 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -507,7 +507,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD String reason = shorten(e.getMessage()); if (currentResult != null) currentResult.recordFailed(reason); - acks.add(DownSyncRecordAck.retryable(centralID, null, reason)); + acks.add(DownSyncRecordAck.failure(centralID, null, reason)); logger.error("Down-sync failed for record " + centralID + " of " + tableDetail.getTableName() + " : " + e.getMessage(), e); } From 8009ccc52276c715def93b1560f6e98893f1356a Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Fri, 4 Sep 2026 13:34:03 +0530 Subject: [PATCH 59/81] fix: sonarqube comments --- .../dataSyncActivity/DataSyncRepository.java | 59 +++++++--- .../DataSyncRepositoryCentralDownload.java | 65 +++++++---- .../validator/SqlIdentifierValidator.java | 103 ++++++++++++++++++ 3 files changed, 190 insertions(+), 37 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 73eb47a8..9fef4ece 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -41,6 +41,7 @@ import org.springframework.stereotype.Service; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo; +import com.iemr.mmu.utils.validator.SqlIdentifierValidator; /*** * @@ -193,8 +194,15 @@ public Map getLocalRecordForDownSync(String schema, String table return null; jdbcTemplate = getJdbcTemplate(); - String query = " SELECT " + autoIncColumnName + ", Processed, " + lastModColumn - + " AS LastModDate, LastDownSyncDate FROM " + schema + "." + table + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + String validLastModColumn = SqlIdentifierValidator.validatedColumnName(lastModColumn); + + String query = " SELECT " + validPkColumn + ", Processed, " + validLastModColumn + + " AS LastModDate, LastDownSyncDate FROM " + validSchema + "." + validTable + " WHERE CentralID = ? AND VanID = ? "; List> resultSet = jdbcTemplate.queryForList(query, centralID, vanID); @@ -227,7 +235,14 @@ public int updateDownSyncRecordInLocal(String query, Object[] params) { public int updateVanSerialNoInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); - String query = " UPDATE " + schema + "." + table + " SET VanSerialNo = ? WHERE " + autoIncColumnName + " = ? "; + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + + String query = " UPDATE " + validSchema + "." + validTable + " SET VanSerialNo = ? WHERE " + + validPkColumn + " = ? "; return jdbcTemplate.update(query, localID, localID); } @@ -238,24 +253,35 @@ public Long resolveLocalIdForCentralValue(String schema, String table, String pk return null; jdbcTemplate = getJdbcTemplate(); - String from = " FROM " + schema + "." + table + " WHERE "; - - Long resolved = queryForFirstLong(" SELECT " + pkColumn + from + " CentralID = ? AND VanID = ? ", centralValue, - vanID); + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumn); + String from = " FROM " + validSchema + "." + validTable + " WHERE "; + + Long resolved = queryForFirstLong(" SELECT " + validPkColumn + from + " CentralID = ? AND VanID = ? ", + centralValue, vanID); if (resolved != null) return resolved; - resolved = queryForFirstLong(" SELECT " + pkColumn + from + " VanSerialNo = ? AND VanID = ? ", centralValue, - vanID); + resolved = queryForFirstLong(" SELECT " + validPkColumn + from + " VanSerialNo = ? AND VanID = ? ", + centralValue, vanID); if (resolved != null) return resolved; - return queryForFirstLong(" SELECT " + pkColumn + from + pkColumn + " = ? AND VanID = ? ", centralValue, vanID); + return queryForFirstLong(" SELECT " + validPkColumn + from + validPkColumn + " = ? AND VanID = ? ", + centralValue, vanID); } public int countConflictsInLocal(String schema, String table, Object vanID) { jdbcTemplate = getJdbcTemplate(); - String query = " SELECT COUNT(*) FROM " + schema + "." + table + // schema & table cannot be bound as query parameters, so both are validated + // before they are concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + + String query = " SELECT COUNT(*) FROM " + validSchema + "." + validTable + " WHERE VanID = ? AND Processed = 'F' AND SyncFailureReason = 'CONFLICT' "; Integer count = jdbcTemplate.queryForObject(query, Integer.class, vanID); return count == null ? 0 : count; @@ -302,9 +328,16 @@ private Long queryForFirstLong(String query, Object... params) { public int markDownSyncConflictInLocal(String schema, String table, String autoIncColumnName, Object localID) { jdbcTemplate = getJdbcTemplate(); - String query = " UPDATE " + schema + "." + table + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + + String query = " UPDATE " + validSchema + "." + validTable + " SET Processed = 'F', SyncFailureReason = 'CONFLICT'," - + " DownSynced = 'F', DownSyncFailureReason = 'CONFLICT' WHERE " + autoIncColumnName + " = ? "; + + " DownSynced = 'F', DownSyncFailureReason = 'CONFLICT' WHERE " + validPkColumn + + " = ? "; return jdbcTemplate.update(query, localID); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index ba4692eb..50ea0b09 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -29,6 +29,7 @@ import javax.sql.DataSource; import com.iemr.mmu.data.syncActivity_syncLayer.DownSyncRecordAck; +import com.iemr.mmu.utils.validator.SqlIdentifierValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -232,20 +233,26 @@ public List> getDownSyncDataFromTable(String schema, String if (schema == null || table == null) throw new Exception("Invalid down-sync request. Schema/table info is missing"); - String columns = (columnNames == null || columnNames.trim().isEmpty()) ? " * " : columnNames; + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String columns = (columnNames == null || columnNames.trim().isEmpty()) ? " * " + : SqlIdentifierValidator.validatedColumnList(columnNames); List> resultSetList; if (DOWN_SYNC_TRANSACTIONAL.equalsIgnoreCase(tableType)) { if (vanID == null) - throw new Exception("Invalid down-sync request. VanID is mandatory for transactional table " + table); + throw new Exception( + "Invalid down-sync request. VanID is mandatory for transactional table " + validTable); - boolean paged = pkColumn != null && pkColumn.matches("^[a-zA-Z_][a-zA-Z0-9_]*$") && batchSize != null - && batchSize > 0; + String validLastModColumn = SqlIdentifierValidator.validatedColumnName(lastModColumn); + boolean paged = SqlIdentifierValidator.isValidIdentifier(pkColumn) && batchSize != null && batchSize > 0; - StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(schema) - .append(".").append(table) + StringBuilder query = new StringBuilder(" SELECT ").append(columns).append(" FROM ").append(validSchema) + .append(".").append(validTable) .append(" WHERE VanID = ? AND ( DownSynced IS NULL OR DownSynced IN ('N', 'U', 'F') ") - .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(lastModColumn) + .append(" OR ( DownSynced = 'P' AND DownSyncDate IS NOT NULL AND ").append(validLastModColumn) .append(" > DownSyncDate ) ) ") .append(" AND ( DownSyncFailureReason IS NULL OR DownSyncFailureReason <> '") .append(DownSyncRecordAck.CONFLICT).append("' ) "); @@ -254,22 +261,23 @@ public List> getDownSyncDataFromTable(String schema, String params.add(vanID); if (paged) { + String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumn); if (lastFetchedID != null) { - query.append(" AND ").append(pkColumn).append(" > ? "); + query.append(" AND ").append(validPkColumn).append(" > ? "); params.add(lastFetchedID); } - query.append(" ORDER BY ").append(pkColumn).append(" LIMIT ").append(batchSize.intValue()); + query.append(" ORDER BY ").append(validPkColumn).append(" LIMIT ").append(batchSize.intValue()); } - logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + logger.info("Down-sync select query for {}.{} : {}", validSchema, validTable, query); resultSetList = jdbcTemplate.queryForList(query.toString(), params.toArray()); } else { - String query = " SELECT " + columns + " FROM " + schema + "." + table; - logger.info("Down-sync select query for {}.{} : {}", schema, table, query); + String query = " SELECT " + columns + " FROM " + validSchema + "." + validTable; + logger.info("Down-sync select query for {}.{} : {}", validSchema, validTable, query); resultSetList = jdbcTemplate.queryForList(query); } - logger.info("Down-sync record count for {}.{} : {}", schema, table, resultSetList.size()); + logger.info("Down-sync record count for {}.{} : {}", validSchema, validTable, resultSetList.size()); return resultSetList; } @@ -282,6 +290,12 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk if (records == null || records.isEmpty()) return 0; + // schema, table & primary key column cannot be bound as query parameters, so + // each of them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumnName); + List successWithSerialNo = new ArrayList<>(); List successWithoutSerialNo = new ArrayList<>(); List failed = new ArrayList<>(); @@ -306,33 +320,33 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk int updatedRows = 0; if (!successWithSerialNo.isEmpty()) { - String query = " UPDATE " + schema + "." + table + String query = " UPDATE " + validSchema + "." + validTable + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL, VanSerialNo = ? " - + " WHERE " + pkColumnName + " = ? "; + + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithSerialNo)); } if (!successWithoutSerialNo.isEmpty()) { - String query = " UPDATE " + schema + "." + table + String query = " UPDATE " + validSchema + "." + validTable + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE " - + pkColumnName + " = ? "; + + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); } if (!retryable.isEmpty()) { - String query = " UPDATE " + schema + "." + table - + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + String query = " UPDATE " + validSchema + "." + validTable + + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, retryable)); } if (!failed.isEmpty()) { - String query = " UPDATE " + schema + "." + table - + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + pkColumnName + " = ? "; + String query = " UPDATE " + validSchema + "." + validTable + + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, failed)); } - logger.info("Down-sync flag updated for {}.{}. Records : {}, rows : {}", schema, table, records.size(), - updatedRows); + logger.info("Down-sync flag updated for {}.{}. Records : {}, rows : {}", validSchema, validTable, + records.size(), updatedRows); return updatedRows; } @@ -341,7 +355,10 @@ public int markDownSyncedPostUpSync(String schema, String table, List return 0; jdbcTemplate = getJdbcTemplate(); - String query = " UPDATE " + schema + "." + table + // schema & table cannot be bound as query parameters, so both are validated + // before they are concatenated into the query + String query = " UPDATE " + SqlIdentifierValidator.validatedSchemaName(schema) + "." + + SqlIdentifierValidator.validatedTableName(table) + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE VanSerialNo = ? AND VanID = ? "; return countUpdatedRows(jdbcTemplate.batchUpdate(query, vanSerialNoAndVanID)); diff --git a/src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java b/src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java new file mode 100644 index 00000000..ed058a30 --- /dev/null +++ b/src/main/java/com/iemr/mmu/utils/validator/SqlIdentifierValidator.java @@ -0,0 +1,103 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +package com.iemr.mmu.utils.validator; + +import java.util.Set; +import java.util.regex.Pattern; + +/*** + * Validates the schema, table and column names that the data-sync layer has to + * concatenate into its dynamic SQL, because a schema/table/column name cannot be + * supplied as a prepared-statement parameter. Every such identifier reaching a + * query must first be passed through this class, so that nothing but a plain SQL + * identifier can ever become part of a statement. + */ +public final class SqlIdentifierValidator { + + private static final Pattern IDENTIFIER_PATTERN = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_]*$"); + + private static final int MAX_IDENTIFIER_LENGTH = 64; + + private static final Set VALID_SCHEMAS = Set.of("public", "db_iemr", "db_identity", "apl_db_iemr", + "apl_db_identity", "db_iemr_sync", "db_identity_sync"); + + private SqlIdentifierValidator() { + // utility class + } + + public static boolean isValidIdentifier(String identifier) { + return identifier != null && identifier.length() <= MAX_IDENTIFIER_LENGTH + && IDENTIFIER_PATTERN.matcher(identifier).matches(); + } + + /*** + * Returns the allow-listed constant matching the given schema name, so that the + * value concatenated into a query never originates from the caller. + */ + public static String validatedSchemaName(String schemaName) { + if (schemaName != null) { + for (String allowedSchema : VALID_SCHEMAS) { + if (allowedSchema.equalsIgnoreCase(schemaName)) { + return allowedSchema; + } + } + } + throw new IllegalArgumentException("Invalid schema name provided for the data-sync query"); + } + + public static String validatedIdentifier(String identifier, String identifierType) { + if (!isValidIdentifier(identifier)) { + throw new IllegalArgumentException( + "Invalid " + identifierType + " provided for the data-sync query"); + } + return identifier; + } + + public static String validatedTableName(String tableName) { + return validatedIdentifier(tableName, "table name"); + } + + public static String validatedColumnName(String columnName) { + return validatedIdentifier(columnName, "column name"); + } + + /*** + * Validates a comma separated column list, as used in the SELECT clause of the + * sync queries. Returns the trimmed, comma separated list built back from the + * validated names. + */ + public static String validatedColumnList(String columnNames) { + if (columnNames == null || columnNames.trim().isEmpty()) { + throw new IllegalArgumentException("Invalid column list provided for the data-sync query"); + } + + StringBuilder validatedColumns = new StringBuilder(); + for (String columnName : columnNames.split(",")) { + if (validatedColumns.length() > 0) { + validatedColumns.append(", "); + } + validatedColumns.append(validatedColumnName(columnName.trim())); + } + return validatedColumns.toString(); + } +} From df18a20b822243a38b78bc24b3c50eb59bec4898 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 14 Jul 2026 16:09:44 +0530 Subject: [PATCH 60/81] fix: add clinical staff information --- .../data/nurse/BeneficiaryVisitDetail.java | 48 +++++++++++++++++++ .../iemr/mmu/repo/login/UserLoginRepo.java | 3 ++ .../mmu/repo/nurse/BenVisitDetailRepo.java | 12 +++++ .../transaction/CommonDoctorServiceImpl.java | 34 +++++++++++++ .../transaction/CommonNurseServiceImpl.java | 20 ++++++++ .../LabTechnicianServiceImpl.java | 22 +++++++++ 6 files changed, 139 insertions(+) diff --git a/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java b/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java index dd17ef7f..ff7778ec 100644 --- a/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java +++ b/src/main/java/com/iemr/mmu/data/nurse/BeneficiaryVisitDetail.java @@ -141,6 +141,22 @@ public class BeneficiaryVisitDetail { @Column(name = "VisitFlowStatusFlag", insertable = false) private String visitFlowStatusFlag; + @Expose + @Column(name = "NurseID") + private Long nurseID; + + @Expose + @Column(name = "DoctorID") + private Long doctorID; + + @Expose + @Column(name = "PharmacistID") + private Long pharmacistID; + + @Expose + @Column(name = "LabTechnicianID") + private Long labTechnicianID; + @Expose @Column(name = "VanSerialNo") private Long vanSerialNo; @@ -535,4 +551,36 @@ public void setFiles(ArrayList> files) { this.files = files; } + public Long getNurseID() { + return nurseID; + } + + public void setNurseID(Long nurseID) { + this.nurseID = nurseID; + } + + public Long getDoctorID() { + return doctorID; + } + + public void setDoctorID(Long doctorID) { + this.doctorID = doctorID; + } + + public Long getPharmacistID() { + return pharmacistID; + } + + public void setPharmacistID(Long pharmacistID) { + this.pharmacistID = pharmacistID; + } + + public Long getLabTechnicianID() { + return labTechnicianID; + } + + public void setLabTechnicianID(Long labTechnicianID) { + this.labTechnicianID = labTechnicianID; + } + } diff --git a/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java b/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java index 6f6390ee..d14e3c3f 100644 --- a/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java +++ b/src/main/java/com/iemr/mmu/repo/login/UserLoginRepo.java @@ -15,6 +15,9 @@ public interface UserLoginRepo extends CrudRepository { @Query(" SELECT u FROM Users u WHERE u.userID = :userID AND u.deleted = false ") public Users getUserByUserID(@Param("userID") Long userID); + @Query(" SELECT u FROM Users u WHERE u.userName = :UserName AND u.deleted = false ") + public Users getUserByUsername(@Param("UserName") String username); + @Query(nativeQuery = true,value = "select rolename from m_role where roleid in (select roleid from m_userservicerolemapping where userid=:userID)") List getRoleNamebyUserId(@Param("userID") Long userID); diff --git a/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java b/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java index c4a03286..75d075f9 100644 --- a/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java +++ b/src/main/java/com/iemr/mmu/repo/nurse/BenVisitDetailRepo.java @@ -101,4 +101,16 @@ public int updateFileID(@Param("fileIDs") String fileIDs, @Param("regID") Long r @Query(" UPDATE BeneficiaryVisitDetail set vanSerialNo = :benVisitID WHERE benVisitID = :benVisitID") int updateVanSerialNo(@Param("benVisitID") Long benVisitID); + // store responsible doctor's user ID against the visit + @Transactional + @Modifying + @Query("UPDATE BeneficiaryVisitDetail set doctorID = :doctorID where visitCode = :visitCode ") + public Integer updateDoctorID(@Param("doctorID") Long doctorID, @Param("visitCode") Long visitCode); + + // store responsible lab technician's user ID against the visit + @Transactional + @Modifying + @Query("UPDATE BeneficiaryVisitDetail set labTechnicianID = :labTechnicianID where visitCode = :visitCode ") + public Integer updateLabTechnicianID(@Param("labTechnicianID") Long labTechnicianID, @Param("visitCode") Long visitCode); + } diff --git a/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java b/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java index cab7a2d3..7302abc4 100644 --- a/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/common/transaction/CommonDoctorServiceImpl.java @@ -115,6 +115,34 @@ public class CommonDoctorServiceImpl { @Autowired private CookieUtil cookieUtil; + @Autowired + private com.iemr.mmu.repo.nurse.BenVisitDetailRepo benVisitDetailRepo; + @Autowired + private com.iemr.mmu.repo.login.UserLoginRepo userLoginRepo; + + /** + * Resolve the numeric user ID of the responsible staff member from the username + * captured in createdBy. Returns null if it cannot be resolved so an unknown + * staff member never blocks the flow. + */ + private Long resolveUserId(String username) { + if (username == null || username.trim().isEmpty()) + return null; + com.iemr.mmu.data.login.Users user = userLoginRepo.getUserByUsername(username.trim()); + return user != null ? user.getUserID() : null; + } + + /** + * Store the responsible doctor's user ID on the visit record, taken from createdBy. + */ + private void storeDoctorIDOnVisit(CommonUtilityClass commonUtilityClass) { + if (commonUtilityClass == null || commonUtilityClass.getVisitCode() == null) + return; + Long doctorID = resolveUserId(commonUtilityClass.getCreatedBy()); + if (doctorID != null) + benVisitDetailRepo.updateDoctorID(doctorID, commonUtilityClass.getVisitCode()); + } + @Autowired public void setSnomedServiceImpl(SnomedServiceImpl snomedServiceImpl) { this.snomedServiceImpl = snomedServiceImpl; @@ -778,6 +806,9 @@ public int updateBenFlowtableAfterDocDataSave(CommonUtilityClass commonUtilityCl Long tmpBenVisitID = commonUtilityClass.getBenVisitID(); Long tmpbeneficiaryRegID = commonUtilityClass.getBeneficiaryRegID(); + // Store the responsible doctor's user ID against the visit + storeDoctorIDOnVisit(commonUtilityClass); + // checking if test is prescribed if (isTestPrescribed) { docFlag = (short) 2; @@ -840,6 +871,9 @@ public int updateBenFlowtableAfterDocDataUpdate(CommonUtilityClass commonUtility Long tmpBenVisitID = commonUtilityClass.getBenVisitID(); Long tmpbeneficiaryRegID = commonUtilityClass.getBeneficiaryRegID(); + // Store the responsible doctor's user ID against the visit + storeDoctorIDOnVisit(commonUtilityClass); + if (commonUtilityClass.getIsSpecialist() != null && commonUtilityClass.getIsSpecialist() == true) { if (isTestPrescribed) tcSpecialistFlag = (short) 2; diff --git a/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java b/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java index 72e3c8f2..a7efa9d4 100644 --- a/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java @@ -145,6 +145,22 @@ public class CommonNurseServiceImpl implements CommonNurseService { private Integer TMReferredWL; @Autowired private BenVisitDetailRepo benVisitDetailRepo; + + @Autowired + private com.iemr.mmu.repo.login.UserLoginRepo userLoginRepo; + + /** + * Resolve the numeric user ID of the responsible staff member from the username + * captured in createdBy. Returns null if it cannot be resolved so an unknown + * staff member never blocks the save. + */ + private Long resolveUserId(String username) { + if (username == null || username.trim().isEmpty()) + return null; + com.iemr.mmu.data.login.Users user = userLoginRepo.getUserByUsername(username.trim()); + return user != null ? user.getUserID() : null; + } + @Autowired private BenChiefComplaintRepo benChiefComplaintRepo; @Autowired @@ -258,6 +274,10 @@ public Long saveBeneficiaryVisitDetails(BeneficiaryVisitDetail beneficiaryVisitD } beneficiaryVisitDetail.setReportFilePath(sb.toString()); + // Store the responsible nurse's user ID (resolved from the createdBy username) + if (beneficiaryVisitDetail.getNurseID() == null) + beneficiaryVisitDetail.setNurseID(resolveUserId(beneficiaryVisitDetail.getCreatedBy())); + response = benVisitDetailRepo.save(beneficiaryVisitDetail); benVisitDetailRepo.updateVanSerialNo(response.getBenVisitID()); diff --git a/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java b/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java index 1026f63c..96f3b5c1 100644 --- a/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/labtechnician/LabTechnicianServiceImpl.java @@ -47,6 +47,22 @@ public class LabTechnicianServiceImpl implements LabTechnicianService { private LabResultEntryRepo labResultEntryRepo; private CommonBenStatusFlowServiceImpl commonBenStatusFlowServiceImpl; + @Autowired + private com.iemr.mmu.repo.nurse.BenVisitDetailRepo benVisitDetailRepo; + @Autowired + private com.iemr.mmu.repo.login.UserLoginRepo userLoginRepo; + + /** + * Resolve the numeric user ID of the responsible staff member from the username + * captured in createdBy. Returns null if it cannot be resolved. + */ + private Long resolveUserId(String username) { + if (username == null || username.trim().isEmpty()) + return null; + com.iemr.mmu.data.login.Users user = userLoginRepo.getUserByUsername(username.trim()); + return user != null ? user.getUserID() : null; + } + @Autowired public void setCommonBenStatusFlowServiceImpl(CommonBenStatusFlowServiceImpl commonBenStatusFlowServiceImpl) { this.commonBenStatusFlowServiceImpl = commonBenStatusFlowServiceImpl; @@ -345,6 +361,12 @@ public Integer saveLabTestResult(JsonObject requestOBJ) throws Exception { labResultSaveFlag = saveLabTestResult(wrapperLabResults); if (labResultSaveFlag == 1) { + // Store the responsible lab technician's user ID against the visit + if (wrapperLabResults.getVisitCode() != null) { + Long labTechnicianID = resolveUserId(wrapperLabResults.getCreatedBy()); + if (labTechnicianID != null) + benVisitDetailRepo.updateLabTechnicianID(labTechnicianID, wrapperLabResults.getVisitCode()); + } int i = updateBenFlowStatusFlagAfterLabResultEntry(wrapperLabResults.getLabCompleted(), wrapperLabResults.getBenFlowID(), wrapperLabResults.getBeneficiaryRegID(), wrapperLabResults.getVisitID(), wrapperLabResults.getNurseFlag(), From 3fc6583d672efd484e59789287e2bcf5694ad124 Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 14 Jul 2026 16:17:10 +0530 Subject: [PATCH 61/81] fix: pom version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6157ff32..7883df9a 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.iemr.mmu mmu-api - 3.8.4 + 3.9.0 war MMU-API From f757174a78ba822f9ec91d5e316dd3273fd86b2e Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Fri, 4 Sep 2026 15:50:50 +0530 Subject: [PATCH 62/81] fix: resolve sonarcube comments --- .../controller/dataSyncActivity/StartSyncActivity.java | 4 ++-- .../dataSyncLayerCentral/MMUDataSyncVanToServer.java | 8 ++++---- .../dataSyncActivity/DownSyncDataFromServerImpl.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 827aded6..daa9b119 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -312,7 +312,7 @@ public String startDownSync(@RequestBody String requestOBJ, response.setError(5000, "vanID is missing, Kindly contact the administrator."); } } catch (Exception e) { - logger.error("Error in down-sync : " + e); + logger.error("Error in down-sync : {}", e.getMessage(), e); response.setError(e); } return response.toString(); @@ -325,7 +325,7 @@ public String checkDownSyncProgress() { try { response.setResponse(new Gson().toJson(downSyncDataFromServerImpl.getDownSyncStatus())); } catch (Exception e) { - logger.error("Error in down-sync progress check : " + e); + logger.error("Error in down-sync progress check : {}", e.getMessage(), e); response.setError(e); } return response.toString(); diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java index 9accb7db..1d2c8790 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncLayerCentral/MMUDataSyncVanToServer.java @@ -198,8 +198,8 @@ public String downSyncDataFromServer(@RequestBody DownSyncDataDigester downSyncD } } catch (Exception e) { if (downSyncDataDigester != null) - logger.error(e + " - Error in down-sync for table " + downSyncDataDigester.getSchemaName() + "." - + downSyncDataDigester.getTableName()); + logger.error("Error in down-sync for table {}.{}", downSyncDataDigester.getSchemaName(), + downSyncDataDigester.getTableName(), e); response.setError(e); } return response.toStringWithSerialization(); @@ -220,8 +220,8 @@ public String updateDownSyncFlagPostDownload(@RequestBody DownSyncDataDigester d } } catch (Exception e) { if (downSyncDataDigester != null) - logger.error(e + " - Error while updating the down-sync flag for " - + downSyncDataDigester.getSchemaName() + "." + downSyncDataDigester.getTableName()); + logger.error("Error while updating the down-sync flag for {}.{}", downSyncDataDigester.getSchemaName(), + downSyncDataDigester.getTableName(), e); response.setError(e); } return response.toStringWithSerialization(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index 795ff80b..e11f1041 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -175,7 +175,7 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer public Map getDownSyncStatus() { Map resultMap = new LinkedHashMap<>(); - resultMap.put("percentage", totalCounter == 0 ? 0 : Math.floor((progressCounter * 100) / totalCounter)); + resultMap.put("percentage", totalCounter == 0 ? 0 : Math.floor((progressCounter * 100.0) / totalCounter)); resultMap.put("inProgress", IN_PROGRESS.get()); resultMap.put("currentTable", currentTable); resultMap.put("totalTables", totalCounter); From 4c621543a1d674ba5fd3882c490f3483f54ee12c Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Wed, 9 Sep 2026 13:57:43 +0530 Subject: [PATCH 63/81] fix: issue in m_beneficiaryregidmapping table --- .../dataSyncActivity/DataSyncRepository.java | 44 ++++++++++++++++--- .../DownSyncDataFromServerImpl.java | 21 ++++++++- .../DataSyncRepositoryCentralDownload.java | 24 +++++++--- 3 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 9fef4ece..75a74842 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -189,7 +189,7 @@ public List getDownSyncColumns(String schema, String table) { } public Map getLocalRecordForDownSync(String schema, String table, String autoIncColumnName, - Object centralID, Object vanID, String lastModColumn) { + Object centralID, Object vanID, String lastModColumn, boolean preserveCentralPK) { if (centralID == null) return null; @@ -201,17 +201,51 @@ public Map getLocalRecordForDownSync(String schema, String table String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); String validLastModColumn = SqlIdentifierValidator.validatedColumnName(lastModColumn); - String query = " SELECT " + validPkColumn + ", Processed, " + validLastModColumn - + " AS LastModDate, LastDownSyncDate FROM " + validSchema + "." + validTable - + " WHERE CentralID = ? AND VanID = ? "; + StringBuilder query = new StringBuilder(" SELECT ").append(validPkColumn).append(", CentralID, Processed, ") + .append(validLastModColumn).append(" AS LastModDate, LastDownSyncDate FROM ").append(validSchema) + .append(".").append(validTable); - List> resultSet = jdbcTemplate.queryForList(query, centralID, vanID); + List params = new ArrayList<>(); + + if (preserveCentralPK) { + query.append(" WHERE ( CentralID = ? OR ").append(validPkColumn).append(" = ? ) "); + params.add(centralID); + params.add(centralID); + } else { + query.append(" WHERE CentralID = ? AND VanID = ? "); + params.add(centralID); + params.add(vanID); + } + + List> resultSet = jdbcTemplate.queryForList(query.toString(), params.toArray()); if (resultSet == null || resultSet.isEmpty()) return null; return resultSet.get(0); } + /*** + * Stamps the central key on a local row that was matched by its primary key, so + * the later runs of the down-sync find it on the CentralID match itself. + */ + public int stampCentralIDInLocal(String schema, String table, String autoIncColumnName, Object localID, + Object centralID) { + if (localID == null || centralID == null) + return 0; + + jdbcTemplate = getJdbcTemplate(); + // schema, table & column names cannot be bound as query parameters, so each of + // them is validated before it is concatenated into the query + String validSchema = SqlIdentifierValidator.validatedSchemaName(schema); + String validTable = SqlIdentifierValidator.validatedTableName(table); + String validPkColumn = SqlIdentifierValidator.validatedColumnName(autoIncColumnName); + + String query = " UPDATE " + validSchema + "." + validTable + " SET CentralID = ? WHERE " + validPkColumn + + " = ? AND CentralID IS NULL "; + + return jdbcTemplate.update(query, centralID, localID); + } + public Long insertDownSyncRecordInLocal(final String query, final Object[] params) { jdbcTemplate = getJdbcTemplate(); KeyHolder keyHolder = new GeneratedKeyHolder(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index e11f1041..c3c1e1d0 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -72,6 +72,11 @@ public class DownSyncDataFromServerImpl implements DownSyncDataFromServer { private static final String PROCESSED = "Processed"; private static final String SYNC_FAILURE_REASON = "SyncFailureReason"; private static final String CENTRAL_ID = "CentralID"; + private static final String PROVISIONED = "Provisioned"; + private static final String RESERVED = "Reserved"; + private static final String RESERVED_FOR = "ReservedFor"; + private static final String RESERVED_BY_ID = "ReservedById"; + private static final String RESERVED_ON = "ReservedOn"; private static final int ACK_BATCH_SIZE = 500; @@ -450,7 +455,7 @@ private List saveTransactionalData(DownSyncTableDetail tableD Map localRecord = dataSyncRepository.getLocalRecordForDownSync( tableDetail.getSchemaName(), tableDetail.getTableName(), pkColumn, centralID, vanID, - lastModColumn); + lastModColumn, tableDetail.isPreserveCentralPK()); // a pointer at another table's primary key means a different number here // than at central, so it is rewritten before the row is written @@ -473,6 +478,11 @@ private List saveTransactionalData(DownSyncTableDetail tableD Long localID = toLong(localRecord.get(pkColumn)); + // a row matched by its primary key alone carries no central key yet + if (localRecord.get(CENTRAL_ID) == null) + dataSyncRepository.stampCentralIDInLocal(tableDetail.getSchemaName(), tableDetail.getTableName(), + pkColumn, localID, centralID); + if (!isCentralCopyNewer(record, localRecord, lastModColumn)) { skippedCounter++; if (currentResult != null) @@ -598,7 +608,8 @@ private void updateRecord(DownSyncTableDetail tableDetail, List serverCo for (int i = 0; i < vanColumns.size(); i++) { String vanColumn = vanColumns.get(i); - if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn)) + if (vanColumn.equalsIgnoreCase(pkColumn) || isDownSyncManagedColumn(vanColumn) + || isVanOwnedColumn(vanColumn)) continue; if (columnsToLeaveAlone != null && containsIgnoringCase(columnsToLeaveAlone, vanColumn)) continue; @@ -816,6 +827,12 @@ private void requireIdentifier(String value, String what, DownSyncTableDetail ta + what); } + private boolean isVanOwnedColumn(String column) { + return VAN_ID.equalsIgnoreCase(column) || PROVISIONED.equalsIgnoreCase(column) + || RESERVED.equalsIgnoreCase(column) || RESERVED_FOR.equalsIgnoreCase(column) + || RESERVED_BY_ID.equalsIgnoreCase(column) || RESERVED_ON.equalsIgnoreCase(column); + } + private boolean isDownSyncManagedColumn(String column) { return VAN_SERIAL_NO.equalsIgnoreCase(column) || CENTRAL_ID.equalsIgnoreCase(column) || PROCESSED.equalsIgnoreCase(column) || SYNC_FAILURE_REASON.equalsIgnoreCase(column) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index 50ea0b09..c76962b4 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -296,6 +296,11 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk String validTable = SqlIdentifierValidator.validatedTableName(table); String validPkColumn = SqlIdentifierValidator.validatedColumnName(pkColumnName); + String lastModColumn = resolveLastModColumn(schema, table); + String holdLastModDate = lastModColumn == null ? "" + : ", " + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " = " + + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " "; + List successWithSerialNo = new ArrayList<>(); List successWithoutSerialNo = new ArrayList<>(); List failed = new ArrayList<>(); @@ -322,26 +327,28 @@ public int updateDownSyncFlagPostDownload(String schema, String table, String pk if (!successWithSerialNo.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL, VanSerialNo = ? " - + " WHERE " + validPkColumn + " = ? "; + + holdLastModDate + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithSerialNo)); } if (!successWithoutSerialNo.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable - + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + " WHERE " - + validPkColumn + " = ? "; + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + holdLastModDate + + " WHERE " + validPkColumn + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, successWithoutSerialNo)); } if (!retryable.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable - + " SET DownSynced = 'U', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; + + " SET DownSynced = 'U', DownSyncFailureReason = ? " + holdLastModDate + " WHERE " + validPkColumn + + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, retryable)); } if (!failed.isEmpty()) { String query = " UPDATE " + validSchema + "." + validTable - + " SET DownSynced = 'F', DownSyncFailureReason = ? " + " WHERE " + validPkColumn + " = ? "; + + " SET DownSynced = 'F', DownSyncFailureReason = ? " + holdLastModDate + " WHERE " + validPkColumn + + " = ? "; updatedRows += countUpdatedRows(jdbcTemplate.batchUpdate(query, failed)); } @@ -357,9 +364,14 @@ public int markDownSyncedPostUpSync(String schema, String table, List jdbcTemplate = getJdbcTemplate(); // schema & table cannot be bound as query parameters, so both are validated // before they are concatenated into the query + String lastModColumn = resolveLastModColumn(schema, table); + String holdLastModDate = lastModColumn == null ? "" + : ", " + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " = " + + SqlIdentifierValidator.validatedColumnName(lastModColumn) + " "; + String query = " UPDATE " + SqlIdentifierValidator.validatedSchemaName(schema) + "." + SqlIdentifierValidator.validatedTableName(table) - + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + + " SET DownSynced = 'P', DownSyncDate = now(), DownSyncFailureReason = NULL " + holdLastModDate + " WHERE VanSerialNo = ? AND VanID = ? "; return countUpdatedRows(jdbcTemplate.batchUpdate(query, vanSerialNoAndVanID)); } From 80304efcaa8312f01790fb57f8d8bafa1c4c11a7 Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Fri, 11 Sep 2026 09:20:02 +0530 Subject: [PATCH 64/81] Adding api's for syncing diagnostic documents to s3 --- .../mmu/controller/dataSyncActivity/StartSyncActivity.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index daa9b119..7e4eb57e 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -70,6 +70,10 @@ public class StartSyncActivity { private DiagnosticDocumentFetchService diagnosticDocumentFetchService; @Autowired private DownSyncDataFromServerImpl downSyncDataFromServerImpl; + @Autowired + private DiagnosticDocumentPushServiceImpl diagnosticDocumentPushServiceImpl; + @Autowired + private DiagnosticDocumentFetchService diagnosticDocumentFetchService; private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; From 5d1182aa1b9b5f3af13de760d479131642320542 Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Fri, 11 Sep 2026 12:15:06 +0530 Subject: [PATCH 65/81] Adding api's for syncing diagnostic documents to s3 --- .../mmu/controller/dataSyncActivity/StartSyncActivity.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 3de7444e..0e9367d6 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -70,12 +70,7 @@ public class StartSyncActivity { private DiagnosticDocumentFetchService diagnosticDocumentFetchService; @Autowired private DownSyncDataFromServerImpl downSyncDataFromServerImpl; - @Autowired - private DiagnosticDocumentPushServiceImpl diagnosticDocumentPushServiceImpl; - @Autowired - private DiagnosticDocumentFetchService diagnosticDocumentFetchService; - @Autowired - private DownSyncDataFromServerImpl downSyncDataFromServerImpl; + private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; From a5f2999ea4a331fec7e839d1631327ac6b12408f Mon Sep 17 00:00:00 2001 From: Sneha Date: Fri, 11 Sep 2026 12:29:26 +0530 Subject: [PATCH 66/81] feat(stoptb): record whether AMRIT caused each Nikshay ID to be created The results-CSV import already wrote the portal's Nikshay ID onto tb_suspected, but nothing distinguished an ID the portal generated for us from one that already existed there and was only matched back. Counting "IDs created by this application" therefore meant inferring it from the ID's presence, which over-counts every pre-existing ID. Each written row now also sets tb_suspected.nikshay_created_by_amrit -- true for a "success" row, false for a "skipped" row (portal-detected duplicate). Rows left for manual review or genuinely failed still write nothing at all, so the existing matching, review and failure behaviour is unchanged; "failed" maps to false for completeness only, as those rows have no ID to write. The import response gains createdByAmrit/alreadyOnNikshay counts, a breakdown of the existing `updated` total that always sums to it, so the figure is available without a query. Additive -- no existing field changes meaning. Requires AMRIT-DB V107 for the column. Note the flag is not yet in the van<->central sync column mapping, so it stays van-local until that row is updated. Co-Authored-By: Claude Opus 5 (1M context) --- .../repo/stoptb/NikshayExportRepository.java | 19 +++++---- .../service/stoptb/NikshayImportService.java | 39 ++++++++++++++++--- 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 28268b3d..81e199bc 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -264,10 +264,14 @@ public Long findLatestSuspectedId(Long benRegId) { return getJdbcTemplate().query(sql, (ResultSet rs) -> rs.next() ? rs.getLong("id") : null, benRegId); } - public void updateNikshayId(Long suspectedId, String nikshayId, String modifiedBy) { - String sql = "UPDATE tb_suspected SET nikshay_id = ?, modified_by = ?, " + /** {@code createdByAmrit} records whether this Nikshay ID was newly + * generated for us by the portal (true) or already existed there and was + * only matched back (false) — see nikshay_created_by_amrit on the write + * methods' class Javadoc. */ + public void updateNikshayId(Long suspectedId, String nikshayId, boolean createdByAmrit, String modifiedBy) { + String sql = "UPDATE tb_suspected SET nikshay_id = ?, nikshay_created_by_amrit = ?, modified_by = ?, " + "last_mod_date = CURRENT_TIMESTAMP WHERE id = ?"; - getJdbcTemplate().update(sql, nikshayId, modifiedBy, suspectedId); + getJdbcTemplate().update(sql, nikshayId, createdByAmrit, modifiedBy, suspectedId); } /** Called when a beneficiary had no tb_suspected row yet — creates one to @@ -275,16 +279,17 @@ public void updateNikshayId(Long suspectedId, String nikshayId, String modifiedB * because, unlike created_date on most other AMRIT tables, tb_suspected's * has no DB-side default. */ public Long insertSuspectedWithNikshayId(Long benRegId, LocalDate visitDate, String nikshayId, - String createdBy) { - String sql = "INSERT INTO tb_suspected (benRegID, visit_date, nikshay_id, created_by, created_date) " - + "VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)"; + boolean createdByAmrit, String createdBy) { + String sql = "INSERT INTO tb_suspected (benRegID, visit_date, nikshay_id, nikshay_created_by_amrit, " + + "created_by, created_date) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)"; KeyHolder keyHolder = new GeneratedKeyHolder(); getJdbcTemplate().update(connection -> { PreparedStatement ps = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); ps.setLong(1, benRegId); ps.setTimestamp(2, Timestamp.valueOf(visitDate.atStartOfDay())); ps.setString(3, nikshayId); - ps.setString(4, createdBy); + ps.setBoolean(4, createdByAmrit); + ps.setString(5, createdBy); return ps; }, keyHolder); return keyHolder.getKey().longValue(); diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java index ebc81e4e..6d3e8d51 100644 --- a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -66,6 +66,18 @@ * as a success; more than one is ambiguous and left for manual review * rather than guessed. * - "failed": never written; surfaced in the response for visibility. + * + * Alongside the ID itself, each written row records + * tb_suspected.nikshay_created_by_amrit — true when the portal generated + * that ID for us ("success"), false when the ID already existed on the + * portal and we only matched it back ("skipped"/"failed"). It exists purely + * so the programme can count how many Nikshay IDs this application actually + * caused to be created, rather than inferring it from the ID's presence. + * Rows written before this column existed, or by any other flow, stay NULL + * — deliberately distinct from an explicit false, since "we don't know" + * isn't the same as "it already existed". Note "failed" rows are never + * written at all (they have no ID), so false in practice comes from + * "skipped"; the mapping covers "failed" for completeness only. */ @Service public class NikshayImportService { @@ -77,7 +89,12 @@ public record ImportRowResult(int rowIndex, Long benRegId, String firstName, Str String status, String generatedId, String note) { } + /** {@code createdByAmrit} + {@code alreadyOnNikshay} are a breakdown of + * {@code updated}: of the rows written, how many carried a portal-generated + * new ID vs. an ID that already existed there. They always sum to + * {@code updated}. */ public record ImportSummary(int csvRowCount, int updated, int failed, int needsReview, + int createdByAmrit, int alreadyOnNikshay, List needsReviewRows, List failedRows) { } @@ -104,6 +121,8 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre } int updated = 0; + int createdByAmritCount = 0; + int alreadyOnNikshayCount = 0; List needsReview = new ArrayList<>(); List failedRows = new ArrayList<>(); @@ -141,8 +160,14 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre if ("success".equalsIgnoreCase(status) || "skipped".equalsIgnoreCase(status)) { String[] tokens = generatedId.isEmpty() ? new String[0] : generatedId.split("\\s+"); if (tokens.length == 1) { - writeNikshayId(visitDate, benRegId, tokens[0], modifiedBy); + boolean createdByAmrit = "success".equalsIgnoreCase(status); + writeNikshayId(visitDate, benRegId, tokens[0], createdByAmrit, modifiedBy); updated++; + if (createdByAmrit) { + createdByAmritCount++; + } else { + alreadyOnNikshayCount++; + } } else { String note = tokens.length == 0 ? "Row marked " + status + " but has no generatedId." : "Multiple possible existing Nikshay IDs (" + generatedId @@ -157,8 +182,8 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre } } - return new ImportSummary(records.size(), updated, failedRows.size(), needsReview.size(), needsReview, - failedRows); + return new ImportSummary(records.size(), updated, failedRows.size(), needsReview.size(), + createdByAmritCount, alreadyOnNikshayCount, needsReview, failedRows); } private static Long parseBenRegId(String raw) { @@ -186,12 +211,14 @@ private static String normalizePhone(String raw) { return digits.matches("[1-9][0-9]{9}") ? digits : null; } - private void writeNikshayId(LocalDate visitDate, Long benRegId, String nikshayId, String modifiedBy) { + private void writeNikshayId(LocalDate visitDate, Long benRegId, String nikshayId, boolean createdByAmrit, + String modifiedBy) { Long suspectedId = nikshayExportRepository.findLatestSuspectedId(benRegId); if (suspectedId != null) { - nikshayExportRepository.updateNikshayId(suspectedId, nikshayId, modifiedBy); + nikshayExportRepository.updateNikshayId(suspectedId, nikshayId, createdByAmrit, modifiedBy); } else { - nikshayExportRepository.insertSuspectedWithNikshayId(benRegId, visitDate, nikshayId, modifiedBy); + nikshayExportRepository.insertSuspectedWithNikshayId(benRegId, visitDate, nikshayId, createdByAmrit, + modifiedBy); } } } From b68a4011ca0c68f459c364797067d868142c5813 Mon Sep 17 00:00:00 2001 From: Sneha Date: Fri, 11 Sep 2026 12:41:55 +0530 Subject: [PATCH 67/81] chore(stoptb): remove added code comments Co-Authored-By: Claude Opus 5 (1M context) --- .../mmu/repo/stoptb/NikshayExportRepository.java | 4 ---- .../mmu/service/stoptb/NikshayImportService.java | 16 ---------------- 2 files changed, 20 deletions(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 81e199bc..2a8089d0 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -264,10 +264,6 @@ public Long findLatestSuspectedId(Long benRegId) { return getJdbcTemplate().query(sql, (ResultSet rs) -> rs.next() ? rs.getLong("id") : null, benRegId); } - /** {@code createdByAmrit} records whether this Nikshay ID was newly - * generated for us by the portal (true) or already existed there and was - * only matched back (false) — see nikshay_created_by_amrit on the write - * methods' class Javadoc. */ public void updateNikshayId(Long suspectedId, String nikshayId, boolean createdByAmrit, String modifiedBy) { String sql = "UPDATE tb_suspected SET nikshay_id = ?, nikshay_created_by_amrit = ?, modified_by = ?, " + "last_mod_date = CURRENT_TIMESTAMP WHERE id = ?"; diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java index 6d3e8d51..faf35e12 100644 --- a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -66,18 +66,6 @@ * as a success; more than one is ambiguous and left for manual review * rather than guessed. * - "failed": never written; surfaced in the response for visibility. - * - * Alongside the ID itself, each written row records - * tb_suspected.nikshay_created_by_amrit — true when the portal generated - * that ID for us ("success"), false when the ID already existed on the - * portal and we only matched it back ("skipped"/"failed"). It exists purely - * so the programme can count how many Nikshay IDs this application actually - * caused to be created, rather than inferring it from the ID's presence. - * Rows written before this column existed, or by any other flow, stay NULL - * — deliberately distinct from an explicit false, since "we don't know" - * isn't the same as "it already existed". Note "failed" rows are never - * written at all (they have no ID), so false in practice comes from - * "skipped"; the mapping covers "failed" for completeness only. */ @Service public class NikshayImportService { @@ -89,10 +77,6 @@ public record ImportRowResult(int rowIndex, Long benRegId, String firstName, Str String status, String generatedId, String note) { } - /** {@code createdByAmrit} + {@code alreadyOnNikshay} are a breakdown of - * {@code updated}: of the rows written, how many carried a portal-generated - * new ID vs. an ID that already existed there. They always sum to - * {@code updated}. */ public record ImportSummary(int csvRowCount, int updated, int failed, int needsReview, int createdByAmrit, int alreadyOnNikshay, List needsReviewRows, List failedRows) { From 3254e125ed09273c2af38b3ff38dd815394a7494 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Mon, 14 Sep 2026 11:01:04 +0530 Subject: [PATCH 68/81] fix(stoptb): store Nikshay ID on i_beneficiarydetails, not tb_suspected MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The import previously fabricated a brand-new tb_suspected row for any beneficiary who didn't already have one, just to hold the generated Nikshay ID. tb_suspected is a clinical table (is_confirmed, is_referred, reason_for_suspicion, etc.) — inserting a bare row there falsely implied the beneficiary had been assessed as a TB suspect when no such assessment ever happened, and every beneficiary needs a Nikshay ID regardless of suspect status. A Nikshay ID is a beneficiary-level identifier, not a clinical fact, so it now lives on db_identity.i_beneficiarydetails (nikshayId, nikshayCreatedByAmrit — new columns, migration to follow separately in AMRIT-DB) alongside similar existing identifiers like rchid on i_beneficiarydetails_rmnch. - NikshayExportRepository: existingNikshayId now reads d.nikshayId (already-joined i_beneficiarydetails) instead of a tb_suspected subquery. Replaced findLatestSuspectedId/updateNikshayId/ insertSuspectedWithNikshayId with a single writeNikshayId() — always an UPDATE via i_beneficiarymapping.BenRegId -> BenDetailsId, never an INSERT, since every beneficiary reaching this point already has a synced i_beneficiarydetails row. - NikshayImportService: calls writeNikshayId() directly and reports a failed row if it updates zero rows, instead of silently dropping the generated ID. tb_suspected is no longer touched by this feature. Co-Authored-By: Claude Sonnet 5 --- .../repo/stoptb/NikshayExportRepository.java | 77 ++++++++----------- .../service/stoptb/NikshayImportService.java | 35 +++++---- 2 files changed, 54 insertions(+), 58 deletions(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 2a8089d0..5302188d 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -24,7 +24,6 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.sql.Timestamp; import java.time.LocalDate; import java.util.List; @@ -35,8 +34,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.PreparedStatementSetter; -import org.springframework.jdbc.support.GeneratedKeyHolder; -import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Repository; /** @@ -89,9 +86,15 @@ * callers can report totals, but there is deliberately no separate report * of *who* was skipped or why. * - * The Nikshay ID itself lives on db_iemr.tb_suspected.nikshay_id — not - * tb_stoptb_diagnostics, which also has a nikshay_id column but is not the - * table this feature writes to. + * The Nikshay ID itself lives on db_identity.i_beneficiarydetails.nikshayId — + * a beneficiary-level identifier, not a clinical fact, so it's stored + * alongside the beneficiary's identity (same table already holds rchid for + * RMNCH), not on db_iemr.tb_suspected. Storing it on tb_suspected would mean + * fabricating a bare "this beneficiary is a TB suspect" row for anyone who + * never actually had a genuine suspected-case row — wrong, since every + * Stop TB beneficiary needs a Nikshay ID regardless of suspect status. + * Not tb_stoptb_diagnostics either, which also has its own nikshay_id column + * but is a separate, manually-entered field this feature never touches. */ @Repository public class NikshayExportRepository { @@ -146,8 +149,7 @@ public record NikshayRawRow(Long benRegId, String firstName, String middleLastNa + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = m.BenRegId " + " AND ge.deleted = 0 ORDER BY ge.id DESC LIMIT 1) AS hivStatus, " + " d.IsHIVPositive AS isHivPos, " - + " (SELECT s.nikshay_id FROM tb_suspected s WHERE s.benRegID = m.BenRegId " - + " AND s.nikshay_id IS NOT NULL ORDER BY s.id DESC LIMIT 1) AS existingNikshayId " + + " d.nikshayId AS existingNikshayId " + "FROM db_identity.i_beneficiarymapping m " + "LEFT JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = m.BenDetailsId AND d.Deleted = 0 " + "LEFT JOIN db_identity.i_beneficiaryaddress a ON a.BenAddressID = m.BenAddressId " @@ -254,40 +256,29 @@ public List findMatchingBeneficiaryIds(String phoneDigits, String firstNam return getJdbcTemplate().query(sql, (rs, rowNum) -> rs.getLong("BenRegId"), phoneDigits, firstName); } - /** The most recent tb_suspected row for this beneficiary, if any — looked - * up live at import time (no export-time snapshot needed, since the - * beneficiary is identified directly from the results CSV's own benRegId - * column). Null if none exists yet. Note: tb_suspected has no `deleted` - * column, unlike most other AMRIT tables. */ - public Long findLatestSuspectedId(Long benRegId) { - String sql = "SELECT id FROM tb_suspected WHERE benRegID = ? ORDER BY id DESC LIMIT 1"; - return getJdbcTemplate().query(sql, (ResultSet rs) -> rs.next() ? rs.getLong("id") : null, benRegId); - } - - public void updateNikshayId(Long suspectedId, String nikshayId, boolean createdByAmrit, String modifiedBy) { - String sql = "UPDATE tb_suspected SET nikshay_id = ?, nikshay_created_by_amrit = ?, modified_by = ?, " - + "last_mod_date = CURRENT_TIMESTAMP WHERE id = ?"; - getJdbcTemplate().update(sql, nikshayId, createdByAmrit, modifiedBy, suspectedId); - } - - /** Called when a beneficiary had no tb_suspected row yet — creates one to - * hold the Nikshay ID the portal generated. created_date is set explicitly - * because, unlike created_date on most other AMRIT tables, tb_suspected's - * has no DB-side default. */ - public Long insertSuspectedWithNikshayId(Long benRegId, LocalDate visitDate, String nikshayId, - boolean createdByAmrit, String createdBy) { - String sql = "INSERT INTO tb_suspected (benRegID, visit_date, nikshay_id, nikshay_created_by_amrit, " - + "created_by, created_date) VALUES (?, ?, ?, ?, ?, CURRENT_TIMESTAMP)"; - KeyHolder keyHolder = new GeneratedKeyHolder(); - getJdbcTemplate().update(connection -> { - PreparedStatement ps = connection.prepareStatement(sql, Statement.RETURN_GENERATED_KEYS); - ps.setLong(1, benRegId); - ps.setTimestamp(2, Timestamp.valueOf(visitDate.atStartOfDay())); - ps.setString(3, nikshayId); - ps.setBoolean(4, createdByAmrit); - ps.setString(5, createdBy); - return ps; - }, keyHolder); - return keyHolder.getKey().longValue(); + /** Writes the Nikshay ID onto the beneficiary's own identity record + * (db_identity.i_beneficiarydetails.nikshayId) instead of tb_suspected — + * see class Javadoc. Always an UPDATE, never an INSERT: every beneficiary + * this method is called for already has an i_beneficiarydetails row (the + * export only ever surfaces/matches beneficiaries whose identity has + * synced — see streamPendingBeneficiaries/findMatchingBeneficiaryIds, + * both of which require d.FirstName to be non-null), so there's no + * "beneficiary has no row yet" case to fabricate a row for. + * + * Resolved via i_beneficiarymapping.BenRegId -> BenDetailsId, not by + * trusting i_beneficiarydetails.BeneficiaryRegID directly — that column + * is a nullable denormalized copy, not the authoritative link, same as + * every other query in this class. + * + * Returns the number of rows updated. Callers should treat 0 as an error + * — it means no synced i_beneficiarydetails row was found for this + * benRegId, which shouldn't happen given the guarantee above, but is + * worth surfacing rather than silently dropping the ID on the floor. */ + public int writeNikshayId(Long benRegId, String nikshayId, boolean createdByAmrit, String modifiedBy) { + String sql = "UPDATE db_identity.i_beneficiarydetails d " + + "JOIN db_identity.i_beneficiarymapping m ON m.BenDetailsId = d.BeneficiaryDetailsId " + + "SET d.nikshayId = ?, d.nikshayCreatedByAmrit = ?, d.ModifiedBy = ?, d.LastModDate = CURRENT_TIMESTAMP " + + "WHERE m.BenRegId = ? AND m.Deleted = 0 AND d.Deleted = 0"; + return getJdbcTemplate().update(sql, nikshayId, createdByAmrit, modifiedBy, benRegId); } } diff --git a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java index faf35e12..1ed1c857 100644 --- a/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java +++ b/src/main/java/com/iemr/mmu/service/stoptb/NikshayImportService.java @@ -85,6 +85,10 @@ public record ImportSummary(int csvRowCount, int updated, int failed, int needsR @Autowired private NikshayExportRepository nikshayExportRepository; + // visitDate is accepted for API/form compatibility with the controller's + // multipart request but no longer used by the write itself — the Nikshay + // ID now lands on i_beneficiarydetails (no visit_date column there), not + // tb_suspected (see NikshayExportRepository.writeNikshayId's Javadoc). public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStream, String modifiedBy) throws Exception { List records; @@ -145,12 +149,23 @@ public ImportSummary importResults(LocalDate visitDate, InputStream csvInputStre String[] tokens = generatedId.isEmpty() ? new String[0] : generatedId.split("\\s+"); if (tokens.length == 1) { boolean createdByAmrit = "success".equalsIgnoreCase(status); - writeNikshayId(visitDate, benRegId, tokens[0], createdByAmrit, modifiedBy); - updated++; - if (createdByAmrit) { - createdByAmritCount++; + int rowsUpdated = nikshayExportRepository.writeNikshayId(benRegId, tokens[0], createdByAmrit, + modifiedBy); + if (rowsUpdated > 0) { + updated++; + if (createdByAmrit) { + createdByAmritCount++; + } else { + alreadyOnNikshayCount++; + } } else { - alreadyOnNikshayCount++; + // Shouldn't happen — every benRegId reaching here was matched via a + // query that already requires a synced i_beneficiarydetails row (see + // NikshayExportRepository.writeNikshayId's Javadoc) — but surface it + // rather than silently dropping the generated ID on the floor. + failedRows.add(new ImportRowResult(i, benRegId, firstName, middleLastName, status, + generatedId, "No synced beneficiary details found for benRegId " + benRegId + + " — could not write the Nikshay ID.")); } } else { String note = tokens.length == 0 ? "Row marked " + status + " but has no generatedId." @@ -195,14 +210,4 @@ private static String normalizePhone(String raw) { return digits.matches("[1-9][0-9]{9}") ? digits : null; } - private void writeNikshayId(LocalDate visitDate, Long benRegId, String nikshayId, boolean createdByAmrit, - String modifiedBy) { - Long suspectedId = nikshayExportRepository.findLatestSuspectedId(benRegId); - if (suspectedId != null) { - nikshayExportRepository.updateNikshayId(suspectedId, nikshayId, createdByAmrit, modifiedBy); - } else { - nikshayExportRepository.insertSuspectedWithNikshayId(benRegId, visitDate, nikshayId, createdByAmrit, - modifiedBy); - } - } } From e6e770cbf84a5a360571236396a5527cc9ed160e Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Tue, 15 Sep 2026 17:00:17 +0530 Subject: [PATCH 69/81] Dropping duplicate columns in table tb_stoptb_general_examination --- .../com/iemr/mmu/repo/stoptb/NikshayExportRepository.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java index 28268b3d..2c525e5b 100644 --- a/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java +++ b/src/main/java/com/iemr/mmu/repo/stoptb/NikshayExportRepository.java @@ -143,8 +143,10 @@ public record NikshayRawRow(Long benRegId, String firstName, String middleLastNa + " d.incomeStatus AS socioeconomicStatus, " + " (SELECT o.chief_complaint FROM tb_stoptb_general_opd o WHERE o.ben_reg_id = m.BenRegId " + " AND o.deleted = 0 ORDER BY o.id DESC LIMIT 1) AS chiefComplaint, " - + " (SELECT ge.hiv_status FROM tb_stoptb_general_examination ge WHERE ge.beneficiary_reg_id = m.BenRegId " - + " AND ge.deleted = 0 ORDER BY ge.id DESC LIMIT 1) AS hivStatus, " + // hiv_status used to live on tb_stoptb_general_examination too, but that copy was + // dropped (AMRIT-DB V109) once tb_screening became the single source of truth. + + " (SELECT s.hiv_status FROM tb_screening s WHERE s.ben_reg_id = m.BenRegId " + + " AND (s.deleted = 0 OR s.deleted IS NULL) ORDER BY s.id DESC LIMIT 1) AS hivStatus, " + " d.IsHIVPositive AS isHivPos, " + " (SELECT s.nikshay_id FROM tb_suspected s WHERE s.benRegID = m.BenRegId " + " AND s.nikshay_id IS NOT NULL ORDER BY s.id DESC LIMIT 1) AS existingNikshayId " From 71f3f630ecdefefa44f7acdeff9661c7ddfb5fbd Mon Sep 17 00:00:00 2001 From: Vanitha S <116701245+vanitha1822@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:01:19 +0530 Subject: [PATCH 70/81] Merge 3.4.2 with 3.9.0 (#193) * fix: Added proper space in the string * Fix the issue in saving ANC Category Beneficiary (#146) * fix: anc care issue * fix: alignment * fix: datasync issue * fix: datasync issue * fix: sync timeout issue * Release 3.6.0 to Release-3.6.1 (#159) * fix: amm-2147 anc care visit is not saving in the nurse section * fix: amm-2147 anc care visit is not saving in the nurse section * Revert "Sn/3.6.0" * fix: amm-2147 anc care visit is not saving in the nurse section * docs: update README and sync package version with pom (#169) --------- Co-authored-by: SnehaRH Co-authored-by: SnehaRH <77656297+snehar-nd@users.noreply.github.com> Co-authored-by: Arpit Tripathi Co-authored-by: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> --- README.md | 6 +- package.json | 2 +- pom.xml | 2 + .../com/iemr/mmu/data/anc/ANCCareDetails.java | 1 + .../mmu/data/anc/ANCWomenVaccineDetail.java | 1 + .../iemr/mmu/repo/nurse/anc/ANCCareRepo.java | 77 ++++++++++--------- .../repo/nurse/anc/ANCWomenVaccineRepo.java | 56 +++++++------- .../mmu/service/anc/ANCNurseServiceImpl.java | 75 +++++++++--------- .../transaction/CommonNurseServiceImpl.java | 2 +- .../GetDataFromVanAndSyncToDBImpl.java | 2 +- src/main/resources/application.properties | 13 +--- 11 files changed, 120 insertions(+), 117 deletions(-) diff --git a/README.md b/README.md index b9176dec..bffbfc0a 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,10 @@ This microservice is developed using Java and the Spring Boot framework, with My Ensure that the following prerequisites are met before building the MMU service: -* JDK 1.8 +* JDK 17 * Maven * Redis -* Spring Boot v2 +* Spring Boot v3 * MySQL ### Installation @@ -47,7 +47,7 @@ To install the MMU module, please follow these steps: 4. Run the development server: - Start the Redis server. - Run the command `mvn spring-boot:run -DENV_VAR=local`. -5. Open your browser and access `http://localhost:8080/swagger-ui.html#!/` to view the Swagger API documentation. +5. Open your browser and access `http://localhost:8087/swagger-ui/index.html` to view the Swagger API documentation. ## Setting Up Commit Hooks diff --git a/package.json b/package.json index 78f95d7d..f8903554 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mmu-api", - "version": "1.0.0", + "version": "3.6.1", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "prepare": "husky", diff --git a/pom.xml b/pom.xml index f577db19..dd4e04a0 100644 --- a/pom.xml +++ b/pom.xml @@ -37,6 +37,8 @@ reuseReports ${project.basedir}/../target/jacoco.exec java + true + diff --git a/src/main/java/com/iemr/mmu/data/anc/ANCCareDetails.java b/src/main/java/com/iemr/mmu/data/anc/ANCCareDetails.java index 36ba14ed..6733d848 100644 --- a/src/main/java/com/iemr/mmu/data/anc/ANCCareDetails.java +++ b/src/main/java/com/iemr/mmu/data/anc/ANCCareDetails.java @@ -185,6 +185,7 @@ public class ANCCareDetails { public ANCCareDetails() { super(); + this.processed = "N"; // TODO Auto-generated constructor stub } diff --git a/src/main/java/com/iemr/mmu/data/anc/ANCWomenVaccineDetail.java b/src/main/java/com/iemr/mmu/data/anc/ANCWomenVaccineDetail.java index c20e7f89..4bfefa01 100644 --- a/src/main/java/com/iemr/mmu/data/anc/ANCWomenVaccineDetail.java +++ b/src/main/java/com/iemr/mmu/data/anc/ANCWomenVaccineDetail.java @@ -130,6 +130,7 @@ public class ANCWomenVaccineDetail { public ANCWomenVaccineDetail() { super(); + this.processed = "N"; // TODO Auto-generated constructor stub } diff --git a/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCCareRepo.java b/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCCareRepo.java index f4632d66..46219dee 100644 --- a/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCCareRepo.java +++ b/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCCareRepo.java @@ -18,7 +18,7 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see https://www.gnu.org/licenses/. -*/ + */ package com.iemr.mmu.repo.nurse.anc; import java.sql.Date; @@ -37,43 +37,48 @@ @Repository public interface ANCCareRepo extends CrudRepository { - @Query(" SELECT ID, beneficiaryRegID, benVisitID, providerServiceMapID, visitCode, visitNo, comolaintType, duration, description, " - + " lastMenstrualPeriod_LMP, gestationalAgeOrPeriodofAmenorrhea_POA, trimesterNumber, expectedDateofDelivery, " - + "primiGravida, gravida_G, termDeliveries_T, pretermDeliveries_P, abortions_A, livebirths_L, bloodGroup, stillBirth " - + "from ANCCareDetails ba WHERE ba.beneficiaryRegID = :benRegID AND visitCode = :visitCode") - public ArrayList getANCCareDetails(@Param("benRegID") Long benRegID, @Param("visitCode") Long visitCode); + @Query(" SELECT ID, beneficiaryRegID, benVisitID, providerServiceMapID, visitCode, visitNo, comolaintType, duration, description, " + + " lastMenstrualPeriod_LMP, gestationalAgeOrPeriodofAmenorrhea_POA, trimesterNumber, expectedDateofDelivery, " + + "primiGravida, gravida_G, termDeliveries_T, pretermDeliveries_P, abortions_A, livebirths_L, bloodGroup, stillBirth " + + "from ANCCareDetails ba WHERE ba.beneficiaryRegID = :benRegID AND visitCode = :visitCode") + public ArrayList getANCCareDetails(@Param("benRegID") Long benRegID, @Param("visitCode") Long visitCode); - @Query(" SELECT processed from ANCCareDetails where beneficiaryRegID=:benRegID AND visitCode = :visitCode") - public String getBenANCCareDetailsStatus(@Param("benRegID") Long benRegID, @Param("visitCode") Long visitCode); + @Query(" SELECT processed from ANCCareDetails where beneficiaryRegID=:benRegID AND visitCode = :visitCode") + public String getBenANCCareDetailsStatus(@Param("benRegID") Long benRegID, @Param("visitCode") Long visitCode); - @Transactional - @Modifying - @Query("update ANCCareDetails set comolaintType=:comolaintType, duration=:duration, description=:description, " - + "lastMenstrualPeriod_LMP=:lastMenstrualPeriod_LMP, " - + "gestationalAgeOrPeriodofAmenorrhea_POA=:gestationalAgeOrPeriodofAmenorrhea_POA," - + "trimesterNumber=:trimesterNumber, expectedDateofDelivery=:expectedDateofDelivery, primiGravida=:primiGravida, " - + "gravida_G=:gravida_G, termDeliveries_T=:termDeliveries_T, pretermDeliveries_P=:pretermDeliveries_P," - + "abortions_A=:abortions_A,livebirths_L=:livebirths_L, bloodGroup=:bloodGroup, modifiedBy=:modifiedBy, processed=:processed, " - + " stillBirth =:stillBirth " + " where visitCode=:visitCode AND beneficiaryRegID=:beneficiaryRegID") - public int updateANCCareDetails(@Param("comolaintType") String comolaintType, @Param("duration") String duration, - @Param("description") String description, @Param("lastMenstrualPeriod_LMP") Date lastMenstrualPeriod_LMP, - @Param("gestationalAgeOrPeriodofAmenorrhea_POA") Short gestationalAgeOrPeriodofAmenorrhea_POA, - @Param("trimesterNumber") Short trimesterNumber, - @Param("expectedDateofDelivery") Date expectedDateofDelivery, @Param("primiGravida") Boolean primiGravida, - @Param("gravida_G") Short gravida_G, @Param("termDeliveries_T") Short termDeliveries_T, - @Param("pretermDeliveries_P") Short pretermDeliveries_P, @Param("abortions_A") Short abortions_A, - @Param("livebirths_L") Short livebirths_L, @Param("bloodGroup") String bloodGroup, - @Param("modifiedBy") String modifiedBy, @Param("processed") String processed, - @Param("beneficiaryRegID") Long beneficiaryRegID, @Param("visitCode") Long visitCode, - @Param("stillBirth") Integer stillBirth); + @Transactional + @Modifying + @Query("update ANCCareDetails set comolaintType=:comolaintType, duration=:duration, description=:description, " + + "lastMenstrualPeriod_LMP=:lastMenstrualPeriod_LMP, " + + "gestationalAgeOrPeriodofAmenorrhea_POA=:gestationalAgeOrPeriodofAmenorrhea_POA," + + "trimesterNumber=:trimesterNumber, expectedDateofDelivery=:expectedDateofDelivery, primiGravida=:primiGravida, " + + "gravida_G=:gravida_G, termDeliveries_T=:termDeliveries_T, pretermDeliveries_P=:pretermDeliveries_P," + + "abortions_A=:abortions_A,livebirths_L=:livebirths_L, bloodGroup=:bloodGroup, modifiedBy=:modifiedBy, processed=:processed, " + + " stillBirth =:stillBirth " + " where visitCode=:visitCode AND beneficiaryRegID=:beneficiaryRegID") + public int updateANCCareDetails(@Param("comolaintType") String comolaintType, @Param("duration") String duration, + @Param("description") String description, @Param("lastMenstrualPeriod_LMP") Date lastMenstrualPeriod_LMP, + @Param("gestationalAgeOrPeriodofAmenorrhea_POA") Short gestationalAgeOrPeriodofAmenorrhea_POA, + @Param("trimesterNumber") Short trimesterNumber, + @Param("expectedDateofDelivery") Date expectedDateofDelivery, @Param("primiGravida") Boolean primiGravida, + @Param("gravida_G") Short gravida_G, @Param("termDeliveries_T") Short termDeliveries_T, + @Param("pretermDeliveries_P") Short pretermDeliveries_P, @Param("abortions_A") Short abortions_A, + @Param("livebirths_L") Short livebirths_L, @Param("bloodGroup") String bloodGroup, + @Param("modifiedBy") String modifiedBy, @Param("processed") String processed, + @Param("beneficiaryRegID") Long beneficiaryRegID, @Param("visitCode") Long visitCode, + @Param("stillBirth") Integer stillBirth); - @Query(nativeQuery = true, value = " SELECT t.bloodGroup from t_anccare t " - + " where t.beneficiaryRegID=:benRegID ORDER BY t.lastModDate DESC limit 1 ") - public String getBenANCCareDetailsStatus(@Param("benRegID") Long benRegID); + @Query(nativeQuery = true, value = " SELECT t.bloodGroup from t_anccare t " + + " where t.beneficiaryRegID=:benRegID ORDER BY t.lastModDate DESC limit 1 ") + public String getBenANCCareDetailsStatus(@Param("benRegID") Long benRegID); - @Query(" SELECT a FROM ANCCareDetails a WHERE a.beneficiaryRegID = :benRegID " - + " AND (a.gravida_G > 5 OR a.pretermDeliveries_P > 0 OR a.abortions_A > 0 OR a.stillBirth > 0 " - + " OR (a.bloodGroup = 'A-Ve' OR a.bloodGroup = 'B-Ve' OR a.bloodGroup = 'O-Ve' OR a.bloodGroup = 'AB-Ve'))" - + " AND a.deleted is false ") - public ArrayList getANCCareDataForHRP(@Param("benRegID") Long benRegID); + @Query(" SELECT a FROM ANCCareDetails a WHERE a.beneficiaryRegID = :benRegID " + + " AND (a.gravida_G > 5 OR a.pretermDeliveries_P > 0 OR a.abortions_A > 0 OR a.stillBirth > 0 " + + " OR (a.bloodGroup = 'A-Ve' OR a.bloodGroup = 'B-Ve' OR a.bloodGroup = 'O-Ve' OR a.bloodGroup = 'AB-Ve'))" + + " AND a.deleted is false ") + public ArrayList getANCCareDataForHRP(@Param("benRegID") Long benRegID); + + @Transactional + @Modifying + @Query(" UPDATE ANCCareDetails set vanSerialNo = :ID WHERE ID = :ID") + int updateVanSerialNo(@Param("ID") Long ID); } diff --git a/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCWomenVaccineRepo.java b/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCWomenVaccineRepo.java index 63dfa5e0..988925e4 100644 --- a/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCWomenVaccineRepo.java +++ b/src/main/java/com/iemr/mmu/repo/nurse/anc/ANCWomenVaccineRepo.java @@ -18,7 +18,7 @@ * * You should have received a copy of the GNU General Public License * along with this program. If not, see https://www.gnu.org/licenses/. -*/ + */ package com.iemr.mmu.repo.nurse.anc; import java.sql.Date; @@ -35,29 +35,33 @@ import jakarta.transaction.Transactional; @Repository -public interface ANCWomenVaccineRepo extends CrudRepository{ - - @Query(" SELECT ID, beneficiaryRegID, benVisitID, providerServiceMapID, vaccineName, status, receivedDate, receivedFacilityName, visitCode " - + "from ANCWomenVaccineDetail ba WHERE ba.beneficiaryRegID = :benRegID AND ba.visitCode = :visitCode") - public ArrayList getANCWomenVaccineDetails(@Param("benRegID") Long benRegID, - @Param("visitCode") Long visitCode); - - @Query(" SELECT vaccineName, processed from ANCWomenVaccineDetail where beneficiaryRegID=:benRegID AND visitCode = :visitCode AND deleted = false") - public ArrayList getBenANCWomenVaccineStatus(@Param("benRegID") Long benRegID, - @Param("visitCode") Long visitCode); - - @Transactional - @Modifying - @Query("update ANCWomenVaccineDetail set status=:status, receivedDate=:receivedDate, receivedFacilityName=:receivedFacilityName, " - + "modifiedBy=:modifiedBy, processed=:processed " - + "where visitCode=:visitCode AND beneficiaryRegID=:beneficiaryRegID AND vaccineName=:vaccineName") - public int updateANCImmunizationDetails(@Param("status") String status, - @Param("receivedDate") Date receivedDate, - @Param("receivedFacilityName") String receivedFacilityName, - @Param("modifiedBy") String modifiedBy, - @Param("processed") String processed, - @Param("beneficiaryRegID") Long beneficiaryRegID, - @Param("visitCode") Long visitCode, - @Param("vaccineName") String vaccineName); - +public interface ANCWomenVaccineRepo extends CrudRepository { + + @Query(" SELECT ID, beneficiaryRegID, benVisitID, providerServiceMapID, vaccineName, status, receivedDate, receivedFacilityName, visitCode " + + "from ANCWomenVaccineDetail ba WHERE ba.beneficiaryRegID = :benRegID AND ba.visitCode = :visitCode") + public ArrayList getANCWomenVaccineDetails(@Param("benRegID") Long benRegID, + @Param("visitCode") Long visitCode); + + @Query(" SELECT vaccineName, processed from ANCWomenVaccineDetail where beneficiaryRegID=:benRegID AND visitCode = :visitCode AND deleted = false") + public ArrayList getBenANCWomenVaccineStatus(@Param("benRegID") Long benRegID, + @Param("visitCode") Long visitCode); + + @Transactional + @Modifying + @Query("update ANCWomenVaccineDetail set status=:status, receivedDate=:receivedDate, receivedFacilityName=:receivedFacilityName, " + + "modifiedBy=:modifiedBy, processed=:processed " + + "where visitCode=:visitCode AND beneficiaryRegID=:beneficiaryRegID AND vaccineName=:vaccineName") + public int updateANCImmunizationDetails(@Param("status") String status, + @Param("receivedDate") Date receivedDate, + @Param("receivedFacilityName") String receivedFacilityName, + @Param("modifiedBy") String modifiedBy, + @Param("processed") String processed, + @Param("beneficiaryRegID") Long beneficiaryRegID, + @Param("visitCode") Long visitCode, + @Param("vaccineName") String vaccineName); + + @Transactional + @Modifying + @Query(" UPDATE ANCWomenVaccineDetail set vanSerialNo = :ID WHERE ID = :ID") + int updateVanSerialNo(@Param("ID") Long ID); } diff --git a/src/main/java/com/iemr/mmu/service/anc/ANCNurseServiceImpl.java b/src/main/java/com/iemr/mmu/service/anc/ANCNurseServiceImpl.java index 1abc07bd..8a31c636 100644 --- a/src/main/java/com/iemr/mmu/service/anc/ANCNurseServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/anc/ANCNurseServiceImpl.java @@ -141,50 +141,45 @@ public Integer saveBenInvestigationFromDoc(WrapperBenInvestigationANC wrapperBen } else { r = 1; } - } - return r; - } - - - @Override - public Long saveBenAncCareDetails - (ANCCareDetails ancCareDetailsOBJ) throws ParseException { - Long ancCareSuccessFlag = null; - if (ancCareDetailsOBJ.getLmpDate() != null && !ancCareDetailsOBJ.getLmpDate().isEmpty() - && ancCareDetailsOBJ.getLmpDate().length() >= 10) { - String lmpDate = ancCareDetailsOBJ.getLmpDate().split("T")[0]; - ancCareDetailsOBJ - .setLastMenstrualPeriod_LMP(new Date(new SimpleDateFormat("yyyy-MM-dd").parse(lmpDate).getTime())); - } - if (ancCareDetailsOBJ.getExpDelDt() != null && !ancCareDetailsOBJ.getExpDelDt().isEmpty() - && ancCareDetailsOBJ.getExpDelDt().length() >= 10) { - String edDate = ancCareDetailsOBJ.getExpDelDt().split("T")[0]; - ancCareDetailsOBJ - .setExpectedDateofDelivery(new Date(new SimpleDateFormat("yyyy-MM-dd").parse(edDate).getTime())); - } - ANCCareDetails ancCareDetailsRS = ancCareRepo.save(ancCareDetailsOBJ); - if (ancCareDetailsRS != null) { - ancCareDetailsRS.setVanSerialNo((ancCareDetailsRS.getID())); - ancCareSuccessFlag = ancCareDetailsRS.getID(); - } - return ancCareSuccessFlag; } + return r; + } - @Override - public Long saveAncImmunizationDetails - (WrapperAncImmunization wrapperAncImmunizationOBJ) throws ParseException { - Long successFlag = null; - List ancWomenVaccineDetailList = getANCWomenVaccineDetail(wrapperAncImmunizationOBJ); - List ancWomenVaccineDetailRSList = (List) ancWomenVaccineRepo - .saveAll(ancWomenVaccineDetailList); - if (!ancWomenVaccineDetailRSList.isEmpty()) { - ancWomenVaccineDetailRSList.get(0).setVanSerialNo(ancWomenVaccineDetailRSList.get(0).getID()); - successFlag = ancWomenVaccineDetailRSList.get(0).getID(); - } - return successFlag; + @Override + public Long saveBenAncCareDetails(ANCCareDetails ancCareDetailsOBJ) throws ParseException { + Long ancCareSuccessFlag = null; + if (ancCareDetailsOBJ.getLmpDate() != null && !ancCareDetailsOBJ.getLmpDate().isEmpty() + && ancCareDetailsOBJ.getLmpDate().length() >= 10) { + String lmpDate = ancCareDetailsOBJ.getLmpDate().split("T")[0]; + ancCareDetailsOBJ + .setLastMenstrualPeriod_LMP(new Date(new SimpleDateFormat("yyyy-MM-dd").parse(lmpDate).getTime())); + } + if (ancCareDetailsOBJ.getExpDelDt() != null && !ancCareDetailsOBJ.getExpDelDt().isEmpty() + && ancCareDetailsOBJ.getExpDelDt().length() >= 10) { + String edDate = ancCareDetailsOBJ.getExpDelDt().split("T")[0]; + ancCareDetailsOBJ + .setExpectedDateofDelivery(new Date(new SimpleDateFormat("yyyy-MM-dd").parse(edDate).getTime())); + } + ANCCareDetails ancCareDetailsRS = ancCareRepo.save(ancCareDetailsOBJ); + if (ancCareDetailsRS != null) { + ancCareRepo.updateVanSerialNo(ancCareDetailsRS.getID()); + ancCareSuccessFlag = ancCareDetailsRS.getID(); } + return ancCareSuccessFlag; + } - + @Override + public Long saveAncImmunizationDetails(WrapperAncImmunization wrapperAncImmunizationOBJ) throws ParseException { + Long successFlag = null; + List ancWomenVaccineDetailList = getANCWomenVaccineDetail(wrapperAncImmunizationOBJ); + List ancWomenVaccineDetailRSList = (List) ancWomenVaccineRepo + .saveAll(ancWomenVaccineDetailList); + if (!ancWomenVaccineDetailRSList.isEmpty()) { + ancWomenVaccineRepo.updateVanSerialNo(ancWomenVaccineDetailRSList.get(0).getID()); + successFlag = ancWomenVaccineDetailRSList.get(0).getID(); + } + return successFlag; + } private List getANCWomenVaccineDetail(WrapperAncImmunization wrapperAncImmunizationOBJ) throws ParseException { diff --git a/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java b/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java index a7efa9d4..652a61ee 100644 --- a/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/common/transaction/CommonNurseServiceImpl.java @@ -2963,7 +2963,7 @@ private double getQtyForOneDay(String form, String dose, String frequency) { } } else { if (frequency.equalsIgnoreCase("Single Dose") || frequency.equalsIgnoreCase("Stat Dose")|| - frequency.equalsIgnoreCase("Single Dose Before Food") || frequency.equalsIgnoreCase("Single Dose After Food")) { + frequency.equalsIgnoreCase("Single Dose Before Food") || frequency.equalsIgnoreCase("Single Dose After Food")) { if (form.equalsIgnoreCase("Tablet")) { if (dose.equalsIgnoreCase("Half Tab")) { qtyInOneDay = .5; diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java index 3205007c..c0e508cd 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java @@ -454,7 +454,7 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig List currentRecordValues = new ArrayList<>(); for (String column : serverColumnsList) { - logger.info("Column="+column); + logger.debug("Column="+column); Object value = cleanRecord.get(column.trim()); // CRITICAL FIX: Don't convert null to string if (value == null) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c2e28213..d1f926a2 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -56,13 +56,8 @@ logging.level.org.apache.tomcat.jdbc.pool=DEBUG # Connection timeout - time to establish connection spring.mvc.async.request-timeout=600000 -# HTTP client timeouts for RestTemplate -rest.connection.timeout=60000 -rest.read.timeout=600000 -rest.socket.timeout=600000 - -# ============= SERVER TIMEOUT ============= -# Tomcat connection timeout -server.connection-timeout=600000 -server.tomcat.connection-timeout=600000 +# ============= SERVER TIMEOUT (Undertow) ============= +# Undertow settings (the actual server in use - WildFly/JBoss) +server.undertow.no-request-timeout=600000 +server.undertow.max-http-post-size=-1 From 0747e0599567b578dc42b393b6f7058accd39aae Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Tue, 29 Sep 2026 16:02:33 +0530 Subject: [PATCH 71/81] Adding s3 dependency --- pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 7883df9a..5e63c6ab 100644 --- a/pom.xml +++ b/pom.xml @@ -294,11 +294,10 @@ - org.apache.commons - commons-csv - 1.11.0 + software.amazon.awssdk + s3 + 2.25.60 - org.apache.commons commons-csv From 98e3c88d00c86dd53230fd2fbf437564bee22f80 Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Wed, 30 Sep 2026 16:49:40 +0530 Subject: [PATCH 72/81] Making improvements given by coderabbit --- .../dataSyncActivity/StartSyncActivity.java | 2 +- .../DiagnosticDocumentIngestService.java | 27 +++++--- .../DiagnosticDocumentPushServiceImpl.java | 61 +++++++------------ 3 files changed, 40 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 0e9367d6..55884642 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -105,7 +105,7 @@ public String dataSyncToServer(@RequestBody String requestOBJ, @PostMapping(value = { "/diagnostic-documents-to-server" }) public String diagnosticDocumentsToServer(@RequestHeader(value = "Authorization") String authorization, @RequestHeader(value = "ServerAuthorization") String serverAuthorization, - @RequestParam(required = false) Long villageId) { + @RequestParam Long villageId) { OutputResponse response = new OutputResponse(); try { String s = diagnosticDocumentPushServiceImpl.pushPendingDocuments(serverAuthorization, villageId); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java index b8e596b5..507ff235 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java @@ -58,16 +58,18 @@ public String ingestDocuments(String requestOBJ) throws Exception { } private Map ingestOne(Map item) { - Long diagnosticOrderId = asLong(item.get("diagnosticOrderId")); - String externalOrderId = (String) item.get("externalOrderId"); - String documentType = (String) item.get("documentType"); - Map ack = new HashMap<>(); - ack.put("diagnosticOrderId", diagnosticOrderId); - ack.put("externalOrderId", externalOrderId); - ack.put("documentType", documentType); + ack.put("documentId", item.get("documentId")); + ack.put("diagnosticOrderId", item.get("diagnosticOrderId")); + ack.put("externalOrderId", item.get("externalOrderId")); + ack.put("documentType", item.get("documentType")); try { + ack.put("documentId", asLong(item.get("documentId"))); + Long diagnosticOrderId = asLong(item.get("diagnosticOrderId")); + ack.put("diagnosticOrderId", diagnosticOrderId); + String documentType = (String) item.get("documentType"); + byte[] plaintext = Base64.getDecoder().decode((String) item.get("fileContentBase64")); String sha256Hash = (String) item.get("sha256Hash"); @@ -81,6 +83,12 @@ private Map ingestOne(Map item) { Long villageId = asLong(item.get("villageId")); String orderType = (String) item.get("orderType"); String storedFileName = (String) item.get("storedFileName"); + if (villageId == null || beneficiaryId == null || orderType == null || documentType == null + || storedFileName == null) { + ack.put("status", "FAILED"); + ack.put("error", "Missing villageId, beneficiaryId, orderType, documentType or storedFileName"); + return ack; + } String s3Key = villageId + "/" + beneficiaryId + "/" + orderType + "/" + documentType + "/" + storedFileName; String contentType = (String) item.get("contentType"); @@ -94,8 +102,9 @@ private Map ingestOne(Map item) { ack.put("status", "SUCCESS"); ack.put("s3Path", s3Key); } catch (Exception e) { - logger.error("Error ingesting diagnostic document: diagnosticOrderId=" + diagnosticOrderId - + ", documentType=" + documentType, e); + logger.error("Error ingesting diagnostic document: documentId=" + item.get("documentId") + + ", diagnosticOrderId=" + item.get("diagnosticOrderId") + + ", documentType=" + item.get("documentType"), e); ack.put("status", "FAILED"); ack.put("error", e.getMessage()); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java index adb7714c..9374d93b 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java @@ -76,7 +76,7 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws List> rows = pendingRows.subList(offset, Math.min(offset + batchSize, pendingRows.size())); List> payloadItems = new ArrayList<>(); - Map> rowsByAckKey = new HashMap<>(); + Map> rowsById = new HashMap<>(); for (Map row : rows) { Long rowId = asLong(row.get("id")); String base64Plaintext; @@ -86,10 +86,6 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws String encryptedPayload = new String(Files.readAllBytes(filePath), StandardCharsets.UTF_8); base64Plaintext = cryptoUtil.decrypt(encryptedPayload); } catch (Exception e) { - // File missing/unreadable off the shared filesystem - mark just this row - // failed and move on, rather than letting the exception abort the whole run - // (which would leave every other pending row, in this batch and beyond, - // completely untouched). logger.warn("Skipping diagnostic document push, could not read file off disk: id={}, error={}", rowId, e.getMessage()); diagnosticDocumentRepository.markPushFailed(rowId, @@ -108,6 +104,7 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws String contentType = (String) row.get("content_type"); Map item = new HashMap<>(); + item.put("documentId", rowId); item.put("diagnosticOrderId", diagnosticOrderId); item.put("externalOrderId", externalOrderId); item.put("beneficiaryId", row.get("beneficiary_id")); @@ -124,22 +121,13 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws item.put("villageId", villageId); item.put("fileContentBase64", base64Plaintext); payloadItems.add(item); - rowsByAckKey.put(ackKey(externalOrderId, documentType), row); + rowsById.put(rowId, row); } if (payloadItems.isEmpty()) { - // Every row in this batch already got marked failed above (missing file or - // failed decrypt) - nothing left to send. continue; } - // Counted here, not after the central round-trip below - these documents WERE - // successfully decrypted and queued for sending regardless of whether the central - // server subsequently accepts, rejects, or fails to respond to them. Otherwise a - // batch that decrypted fine but got rejected by the central server (e.g. an expired - // session) would leave totalAttempted at 0, and the caller would see the misleading - // "No documents could be decrypted for push" instead of the real per-row reason - // (already recorded in docSyncFailureReason). totalAttempted += payloadItems.size(); String requestOBJ = new Gson().toJson(payloadItems); @@ -152,7 +140,7 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws if (response == null || !response.hasBody()) { logger.warn("No response body from central server for diagnostic document push, marking batch failed"); - markBatchFailed(rowsByAckKey, "No response body from central server"); + markBatchFailed(rowsById, "No response body from central server"); continue; } @@ -161,7 +149,7 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws JsonElement parsedBody = JsonParser.parseString(response.getBody()); if (!parsedBody.isJsonObject()) { logger.warn("Unexpected response shape from central server for diagnostic document push, marking batch failed"); - markBatchFailed(rowsByAckKey, "Unexpected response shape from central server"); + markBatchFailed(rowsById, "Unexpected response shape from central server"); continue; } JsonObject envelope = parsedBody.getAsJsonObject(); @@ -169,7 +157,7 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws || !envelope.has("data")) { logger.warn("Central server reported failure for diagnostic document push batch, marking batch failed: {}", response.getBody()); - markBatchFailed(rowsByAckKey, "Central server reported failure: " + response.getBody()); + markBatchFailed(rowsById, "Central server reported failure: " + response.getBody()); continue; } @@ -177,43 +165,40 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws }.getType(); acks = new Gson().fromJson(envelope.get("data"), ackListType); if (acks == null) { - markBatchFailed(rowsByAckKey, "Central server returned no acknowledgements"); + markBatchFailed(rowsById, "Central server returned no acknowledgements"); continue; } } catch (Exception e) { logger.error("Error calling central server for diagnostic document push, marking batch failed", e); - markBatchFailed(rowsByAckKey, "Error calling central server: " + e.getMessage()); + markBatchFailed(rowsById, "Error calling central server: " + e.getMessage()); continue; } int batchSuccessCount = 0; - Map> unmatchedRowsByAckKey = new HashMap<>(rowsByAckKey); + Map> unmatchedRowsById = new HashMap<>(rowsById); for (Map ack : acks) { - String key = ackKey((String) ack.get("externalOrderId"), (String) ack.get("documentType")); - Map row = rowsByAckKey.get(key); - if (row == null) { + Long rowId = ack.get("documentId") instanceof Number ? asLong(ack.get("documentId")) : null; + if (rowId == null || unmatchedRowsById.remove(rowId) == null) { continue; } - unmatchedRowsByAckKey.remove(key); if ("SUCCESS".equalsIgnoreCase((String) ack.get("status"))) { - diagnosticDocumentRepository.markPushedToCentral(asLong(row.get("id")), - (String) ack.get("s3Path")); + diagnosticDocumentRepository.markPushedToCentral(rowId, (String) ack.get("s3Path")); batchSuccessCount++; } else { String reason = (String) ack.get("error"); - logger.warn("Central server rejected diagnostic document push: externalOrderId={}, documentType={}, error={}", - ack.get("externalOrderId"), ack.get("documentType"), reason); - diagnosticDocumentRepository.markPushFailed(asLong(row.get("id")), + logger.warn("Central server rejected diagnostic document push: id={}, externalOrderId={}, documentType={}, error={}", + rowId, ack.get("externalOrderId"), ack.get("documentType"), reason); + diagnosticDocumentRepository.markPushFailed(rowId, reason != null ? reason : "Central server rejected the document"); } } - if (!unmatchedRowsByAckKey.isEmpty()) { + if (!unmatchedRowsById.isEmpty()) { // Central sent back fewer acks than documents we sent - whatever wasn't // accounted for must not be silently left at its previous status forever. logger.warn( "Diagnostic document push: {} row(s) in this batch got no matching ack back, marking failed", - unmatchedRowsByAckKey.size()); - markBatchFailed(unmatchedRowsByAckKey, "No acknowledgement received from central server"); + unmatchedRowsById.size()); + markBatchFailed(unmatchedRowsById, "No acknowledgement received from central server"); } totalSucceeded += batchSuccessCount; @@ -239,19 +224,15 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws return "Data successfully synced"; } - private static String ackKey(String externalOrderId, String documentType) { - return externalOrderId + "|" + documentType; - } - /*** * @purpose Called when the whole batch call to the central server fails (no/garbled * response, non-200, or a thrown exception) - marks every row that was in this * batch as failed, matching how /van-to-server marks a whole batch failed on a * connection error, rather than leaving them stuck at docsProcessed='N' forever. */ - private void markBatchFailed(Map> rowsByAckKey, String reason) { - for (Map row : rowsByAckKey.values()) { - diagnosticDocumentRepository.markPushFailed(asLong(row.get("id")), reason); + private void markBatchFailed(Map> rowsById, String reason) { + for (Long rowId : rowsById.keySet()) { + diagnosticDocumentRepository.markPushFailed(rowId, reason); } } From 5af9ca3257922a3acec6cb8987e07e56c2f5f438 Mon Sep 17 00:00:00 2001 From: Sehjot Singh Pannu Date: Thu, 1 Oct 2026 17:07:23 +0530 Subject: [PATCH 73/81] Made changes in pom release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5e63c6ab..fc6a604d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.iemr.mmu mmu-api - 3.9.0 + 3.9.2 war MMU-API From acc420fd1e577ca642ecf57fd394f91212ac8d57 Mon Sep 17 00:00:00 2001 From: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:00:14 +0530 Subject: [PATCH 74/81] Use vanID instead of villageId in diagnostic document S3 path (#197) The S3 key was prefixed with a villageId taken from the request param, so every document in a push run was filed under one village regardless of where it came from. Use each row's own vanID instead; villageId is now optional on /diagnostic-documents-to-server and ignored. Co-authored-by: Claude Opus 5.5 --- .../controller/dataSyncActivity/StartSyncActivity.java | 4 ++-- .../DiagnosticDocumentIngestService.java | 8 ++++---- .../DiagnosticDocumentPushServiceImpl.java | 3 +-- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 55884642..3d1f211e 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -105,10 +105,10 @@ public String dataSyncToServer(@RequestBody String requestOBJ, @PostMapping(value = { "/diagnostic-documents-to-server" }) public String diagnosticDocumentsToServer(@RequestHeader(value = "Authorization") String authorization, @RequestHeader(value = "ServerAuthorization") String serverAuthorization, - @RequestParam Long villageId) { + @RequestParam(required = false) Long villageId) { OutputResponse response = new OutputResponse(); try { - String s = diagnosticDocumentPushServiceImpl.pushPendingDocuments(serverAuthorization, villageId); + String s = diagnosticDocumentPushServiceImpl.pushPendingDocuments(serverAuthorization); if (s != null) response.setResponse(s); else diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java index 507ff235..81fa0acf 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java @@ -80,16 +80,16 @@ private Map ingestOne(Map item) { } Long beneficiaryId = asLong(item.get("beneficiaryId")); - Long villageId = asLong(item.get("villageId")); + Long vanId = asLong(item.get("vanID")); String orderType = (String) item.get("orderType"); String storedFileName = (String) item.get("storedFileName"); - if (villageId == null || beneficiaryId == null || orderType == null || documentType == null + if (vanId == null || beneficiaryId == null || orderType == null || documentType == null || storedFileName == null) { ack.put("status", "FAILED"); - ack.put("error", "Missing villageId, beneficiaryId, orderType, documentType or storedFileName"); + ack.put("error", "Missing vanID, beneficiaryId, orderType, documentType or storedFileName"); return ack; } - String s3Key = villageId + "/" + beneficiaryId + "/" + orderType + "/" + documentType + "/" + String s3Key = vanId + "/" + beneficiaryId + "/" + orderType + "/" + documentType + "/" + storedFileName; String contentType = (String) item.get("contentType"); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java index 9374d93b..6a5a9c00 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java @@ -66,7 +66,7 @@ public class DiagnosticDocumentPushServiceImpl { @Autowired private CryptoUtil cryptoUtil; - public String pushPendingDocuments(String Authorization, Long villageId) throws Exception { + public String pushPendingDocuments(String Authorization) throws Exception { List> pendingRows = diagnosticDocumentRepository.findPendingDocuments(); boolean anyRowsFound = !pendingRows.isEmpty(); int totalAttempted = 0; @@ -118,7 +118,6 @@ public String pushPendingDocuments(String Authorization, Long villageId) throws item.put("vanID", row.get("vanID")); item.put("parkingPlaceID", row.get("parkingPlaceID")); item.put("vanSerialNo", row.get("vanSerialNo")); - item.put("villageId", villageId); item.put("fileContentBase64", base64Plaintext); payloadItems.add(item); rowsById.put(rowId, row); From b89287d59ebab7b91bcb094f2a2675986febd038 Mon Sep 17 00:00:00 2001 From: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:45:58 +0530 Subject: [PATCH 75/81] Vb/diagnostic document sync response (#198) * Use vanID instead of villageId in diagnostic document S3 path The S3 key was prefixed with a villageId taken from the request param, so every document in a push run was filed under one village regardless of where it came from. Use each row's own vanID instead; villageId is now optional on /diagnostic-documents-to-server and ignored. Co-Authored-By: Claude Opus 5.5 * Return van-to-server style summary from diagnostic document push The push always answered "Data successfully synced" when at least one document went through, hiding partial failures. Return the same table-level shape /van-to-server uses (status success/partial/failed, totalRecords, successfulRecords, failedRecords, failureReasons) so the data sync screen can show what failed and why. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../DiagnosticDocumentPushServiceImpl.java | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java index 6a5a9c00..cc817200 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentPushServiceImpl.java @@ -6,6 +6,7 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -68,9 +69,8 @@ public class DiagnosticDocumentPushServiceImpl { public String pushPendingDocuments(String Authorization) throws Exception { List> pendingRows = diagnosticDocumentRepository.findPendingDocuments(); - boolean anyRowsFound = !pendingRows.isEmpty(); - int totalAttempted = 0; int totalSucceeded = 0; + List failureReasons = new ArrayList<>(); for (int offset = 0; offset < pendingRows.size(); offset += batchSize) { List> rows = pendingRows.subList(offset, Math.min(offset + batchSize, pendingRows.size())); @@ -88,13 +88,12 @@ public String pushPendingDocuments(String Authorization) throws Exception { } catch (Exception e) { logger.warn("Skipping diagnostic document push, could not read file off disk: id={}, error={}", rowId, e.getMessage()); - diagnosticDocumentRepository.markPushFailed(rowId, - "Could not read file off shared filesystem: " + e.getMessage()); + markFailed(rowId, "Could not read file off shared filesystem: " + e.getMessage(), failureReasons); continue; } if (base64Plaintext == null) { logger.warn("Skipping diagnostic document push, decrypt failed: id={}", rowId); - diagnosticDocumentRepository.markPushFailed(rowId, "Decrypt failed"); + markFailed(rowId, "Decrypt failed", failureReasons); continue; } @@ -127,8 +126,6 @@ public String pushPendingDocuments(String Authorization) throws Exception { continue; } - totalAttempted += payloadItems.size(); - String requestOBJ = new Gson().toJson(payloadItems); List> acks; try { @@ -139,7 +136,7 @@ public String pushPendingDocuments(String Authorization) throws Exception { if (response == null || !response.hasBody()) { logger.warn("No response body from central server for diagnostic document push, marking batch failed"); - markBatchFailed(rowsById, "No response body from central server"); + markBatchFailed(rowsById, "No response body from central server", failureReasons); continue; } @@ -148,7 +145,7 @@ public String pushPendingDocuments(String Authorization) throws Exception { JsonElement parsedBody = JsonParser.parseString(response.getBody()); if (!parsedBody.isJsonObject()) { logger.warn("Unexpected response shape from central server for diagnostic document push, marking batch failed"); - markBatchFailed(rowsById, "Unexpected response shape from central server"); + markBatchFailed(rowsById, "Unexpected response shape from central server", failureReasons); continue; } JsonObject envelope = parsedBody.getAsJsonObject(); @@ -156,7 +153,7 @@ public String pushPendingDocuments(String Authorization) throws Exception { || !envelope.has("data")) { logger.warn("Central server reported failure for diagnostic document push batch, marking batch failed: {}", response.getBody()); - markBatchFailed(rowsById, "Central server reported failure: " + response.getBody()); + markBatchFailed(rowsById, "Central server reported failure: " + response.getBody(), failureReasons); continue; } @@ -164,12 +161,12 @@ public String pushPendingDocuments(String Authorization) throws Exception { }.getType(); acks = new Gson().fromJson(envelope.get("data"), ackListType); if (acks == null) { - markBatchFailed(rowsById, "Central server returned no acknowledgements"); + markBatchFailed(rowsById, "Central server returned no acknowledgements", failureReasons); continue; } } catch (Exception e) { logger.error("Error calling central server for diagnostic document push, marking batch failed", e); - markBatchFailed(rowsById, "Error calling central server: " + e.getMessage()); + markBatchFailed(rowsById, "Error calling central server: " + e.getMessage(), failureReasons); continue; } @@ -187,8 +184,7 @@ public String pushPendingDocuments(String Authorization) throws Exception { String reason = (String) ack.get("error"); logger.warn("Central server rejected diagnostic document push: id={}, externalOrderId={}, documentType={}, error={}", rowId, ack.get("externalOrderId"), ack.get("documentType"), reason); - diagnosticDocumentRepository.markPushFailed(rowId, - reason != null ? reason : "Central server rejected the document"); + markFailed(rowId, reason != null ? reason : "Central server rejected the document", failureReasons); } } if (!unmatchedRowsById.isEmpty()) { @@ -197,7 +193,7 @@ public String pushPendingDocuments(String Authorization) throws Exception { logger.warn( "Diagnostic document push: {} row(s) in this batch got no matching ack back, marking failed", unmatchedRowsById.size()); - markBatchFailed(unmatchedRowsById, "No acknowledgement received from central server"); + markBatchFailed(unmatchedRowsById, "No acknowledgement received from central server", failureReasons); } totalSucceeded += batchSuccessCount; @@ -205,22 +201,47 @@ public String pushPendingDocuments(String Authorization) throws Exception { batchSuccessCount); } - if (!anyRowsFound) { - return "No pending diagnostic documents to sync"; + if (pendingRows.isEmpty()) { + return "No data to sync"; } - if (totalAttempted == 0) { - return "No documents could be decrypted for push"; + + // Same table-level summary shape as /van-to-server (UploadDataToServerImpl) so the + // data sync screen can read both the same way. + int totalRecords = pendingRows.size(); + int failedRecords = failureReasons.size(); + String status; + if (failedRecords == 0) { + status = "success"; + } else if (totalSucceeded == 0) { + status = "failed"; + } else { + status = "partial"; } - if (totalSucceeded == 0) { - // Documents WERE decrypted and sent, but none were accepted (e.g. the central - // server rejected every batch) - the specific reason for each row is recorded in - // its own docSyncFailureReason, this is just the overall-outcome summary. - return "Documents were sent but none were accepted by the central server"; + + Map finalResponse = new LinkedHashMap<>(); + finalResponse.put("response", + "success".equals(status) ? "Data sync completed successfully" : "Data sync completed with failures"); + finalResponse.put("schemaName", "db_iemr"); + finalResponse.put("tableName", "tb_diagnostic_document"); + finalResponse.put("status", status); + finalResponse.put("totalRecords", totalRecords); + finalResponse.put("successfulRecords", totalSucceeded); + finalResponse.put("failedRecords", failedRecords); + if (!failureReasons.isEmpty()) { + finalResponse.put("failureReasons", failureReasons); } - logger.info("Diagnostic document push complete overall: attempted={}, succeeded={}", totalAttempted, - totalSucceeded); - return "Data successfully synced"; + logger.info("Diagnostic document push complete: {} (Success: {}, Failed: {}, Total: {})", status, + totalSucceeded, failedRecords, totalRecords); + return new Gson().toJson(finalResponse); + } + + /*** + * @purpose Marks one row failed locally and records its reason for the run summary. + */ + private void markFailed(Long rowId, String reason, List failureReasons) { + diagnosticDocumentRepository.markPushFailed(rowId, reason); + failureReasons.add("documentId " + rowId + ": " + reason); } /*** @@ -229,9 +250,9 @@ public String pushPendingDocuments(String Authorization) throws Exception { * batch as failed, matching how /van-to-server marks a whole batch failed on a * connection error, rather than leaving them stuck at docsProcessed='N' forever. */ - private void markBatchFailed(Map> rowsById, String reason) { + private void markBatchFailed(Map> rowsById, String reason, List failureReasons) { for (Long rowId : rowsById.keySet()) { - diagnosticDocumentRepository.markPushFailed(rowId, reason); + markFailed(rowId, reason, failureReasons); } } From 777713f120c2be9dc7e35ea0144afb210bede42a Mon Sep 17 00:00:00 2001 From: vanitha1822 Date: Tue, 6 Oct 2026 21:19:00 +0530 Subject: [PATCH 76/81] fix: update the file extension --- .../DiagnosticDocumentIngestService.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java index 81fa0acf..d04d841f 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DiagnosticDocumentIngestService.java @@ -35,6 +35,7 @@ public class DiagnosticDocumentIngestService { private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); private static final Gson GSON = new Gson(); + private static final String ENCRYPTED_FILE_SUFFIX = ".enc"; @Value("${diagnostic.documents.s3.bucket}") private String bucket; @@ -90,7 +91,7 @@ private Map ingestOne(Map item) { return ack; } String s3Key = vanId + "/" + beneficiaryId + "/" + orderType + "/" + documentType + "/" - + storedFileName; + + objectFileName(storedFileName, (String) item.get("fileExtension")); String contentType = (String) item.get("contentType"); s3Client.putObject( @@ -111,6 +112,18 @@ private Map ingestOne(Map item) { return ack; } + /*** + * @purpose The van stores files encrypted as ".enc", but S3 holds the decrypted + * content - so the object is named with the real extension the van derived from + * its content type (e.g. CAD.enc -> CAD.pdf), not the misleading ".enc". + */ + private static String objectFileName(String storedFileName, String fileExtension) { + String baseName = storedFileName.endsWith(ENCRYPTED_FILE_SUFFIX) + ? storedFileName.substring(0, storedFileName.length() - ENCRYPTED_FILE_SUFFIX.length()) + : storedFileName; + return fileExtension != null && !fileExtension.isEmpty() ? baseName + "." + fileExtension : baseName; + } + private static Long asLong(Object value) { return value == null ? null : ((Number) value).longValue(); } From 6c099023af98aaf8c48d29cf26aeb3bf99b2399c Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Wed, 14 Oct 2026 14:06:05 +0530 Subject: [PATCH 77/81] feat(STOP-457): automatic BenID download for StopTB laptops The laptop now downloads BeneficiaryIDs from central by itself, so StopTB registration no longer depends on someone clicking "Generate BenID". - CentralAuthService: after the operator's normal login, signs in to central once as an API client and keeps only the refresh token (AES-GCM encrypted, key tied to this machine, file 600 under /opt/amrit/data). Each run uses refreshToken -> JWT -> getLoginResponse to get a central session key. No password is stored; a password central rejected is never sent again. - BenIdAutoTopUpService: pool 0 -> 10000, below 2000 -> 1000. IDs received from central are written to a pending file before the local import, so a failed import is retried with the same IDs. Retries 5/15/30/60 min; one lock shared with the manual "Generate BenID" button. - /centralAuth/status and /centralAuth/capture for MMU-UI. - Top-up check also runs after a successful van-to-server upload and master download. - Only active when stoptb.enforce.vanid=true and stoptb.van.id>0; every setting has a default, so other deployments need no config change. No change on central (Common-API / BenGen). Co-Authored-By: Claude Opus 5.5 --- src/main/environment/common_ci.properties | 8 +- src/main/environment/common_docker.properties | 16 +- .../environment/common_example.properties | 16 +- src/main/java/com/iemr/mmu/Application.java | 2 + .../benid/CentralAuthController.java | 96 +++++ .../dataSyncActivity/StartSyncActivity.java | 11 +- .../service/benid/BenIdAutoTopUpService.java | 322 +++++++++++++++ .../mmu/service/benid/CentralAuthService.java | 377 ++++++++++++++++++ .../DownloadDataFromServerImpl.java | 8 + .../benid/BenIdAutoTopUpServiceTest.java | 225 +++++++++++ .../service/benid/CentralAuthServiceTest.java | 245 ++++++++++++ 11 files changed, 1322 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java create mode 100644 src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java create mode 100644 src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java create mode 100644 src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java create mode 100644 src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java diff --git a/src/main/environment/common_ci.properties b/src/main/environment/common_ci.properties index 34875d64..cd33f081 100644 --- a/src/main/environment/common_ci.properties +++ b/src/main/environment/common_ci.properties @@ -106,4 +106,10 @@ logging.file.name=@env.MMU_API_LOGGING_FILE_NAME@ springdoc.api-docs.enabled=@env.SWAGGER_DOC_ENABLED@ springdoc.swagger-ui.enabled=@env.SWAGGER_DOC_ENABLED@ -cors.allowed-origins=@env.CORS_ALLOWED_ORIGINS@ \ No newline at end of file +cors.allowed-origins=@env.CORS_ALLOWED_ORIGINS@ + +## BenID auto top-up URLs (feature stays off: stoptb.* not set for these builds) +benCheckUrlLocal=@env.IDENTITY_API_BASE_URL@id/checkAvailablBenIDLocalServer +centralLoginUrl=@env.COMMON_API_OPEN_SYNC@user/userAuthenticate +centralRefreshTokenUrl=@env.COMMON_API_OPEN_SYNC@user/refreshToken +centralSessionUrl=@env.COMMON_API_OPEN_SYNC@user/getLoginResponse diff --git a/src/main/environment/common_docker.properties b/src/main/environment/common_docker.properties index be5f3608..38be2dce 100644 --- a/src/main/environment/common_docker.properties +++ b/src/main/environment/common_docker.properties @@ -104,4 +104,18 @@ logging.path=logs logging.file.name=${MMU_API_LOGGING_FILE_NAME} springdoc.api-docs.enabled=${SWAGGER_DOC_ENABLED} -springdoc.swagger-ui.enabled=${SWAGGER_DOC_ENABLED} \ No newline at end of file +springdoc.swagger-ui.enabled=${SWAGGER_DOC_ENABLED} + +## BenID auto top-up (StopTB laptops only; off unless STOPTB_ENFORCE_VANID=true and STOPTB_VAN_ID>0) +benCheckUrlLocal=${IDENTITY_API_BASE_URL}id/checkAvailablBenIDLocalServer +centralLoginUrl=${COMMON_API_OPEN_SYNC}user/userAuthenticate +centralRefreshTokenUrl=${COMMON_API_OPEN_SYNC}user/refreshToken +centralSessionUrl=${COMMON_API_OPEN_SYNC}user/getLoginResponse +stoptb.enforce.vanid=${STOPTB_ENFORCE_VANID:false} +stoptb.van.id=${STOPTB_VAN_ID:0} +benid.pool.lowerLimit=${BENID_POOL_LOWER_LIMIT:2000} +benid.pool.batchSize=${BENID_POOL_BATCH_SIZE:1000} +benid.pool.bootstrapSize=${BENID_POOL_BOOTSTRAP_SIZE:10000} +benid.token.dir=${BENID_TOKEN_DIR:/opt/amrit/data} +benid.autoTopUp.initialDelayMs=${BENID_AUTO_TOPUP_INITIAL_DELAY_MS:120000} +benid.autoTopUp.checkIntervalMs=${BENID_AUTO_TOPUP_CHECK_INTERVAL_MS:300000} diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 515bb23c..63dd2fa8 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -120,4 +120,18 @@ logging.path=logs/ logging.file.name=logs/mmu-api.log -cors.allowed-origins=http://localhost:* \ No newline at end of file +cors.allowed-origins=http://localhost:* + +## BenID auto top-up (StopTB laptops only) +benCheckUrlLocal=http://localhost:8094/id/checkAvailablBenIDLocalServer +centralLoginUrl=https://uatstoptb.piramalswasthya.org/common-api/user/userAuthenticate +centralRefreshTokenUrl=https://uatstoptb.piramalswasthya.org/common-api/user/refreshToken +centralSessionUrl=https://uatstoptb.piramalswasthya.org/common-api/user/getLoginResponse +stoptb.enforce.vanid=false +stoptb.van.id=0 +benid.pool.lowerLimit=2000 +benid.pool.batchSize=1000 +benid.pool.bootstrapSize=10000 +benid.token.dir=/opt/amrit/data +benid.autoTopUp.initialDelayMs=120000 +benid.autoTopUp.checkIntervalMs=300000 diff --git a/src/main/java/com/iemr/mmu/Application.java b/src/main/java/com/iemr/mmu/Application.java index 712d34f5..8fb40ccf 100644 --- a/src/main/java/com/iemr/mmu/Application.java +++ b/src/main/java/com/iemr/mmu/Application.java @@ -29,11 +29,13 @@ import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.scheduling.annotation.EnableScheduling; import com.iemr.mmu.data.login.Users; import com.iemr.mmu.utils.IEMRApplBeans; @SpringBootApplication +@EnableScheduling public class Application { public static void main(String[] args) { diff --git a/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java b/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java new file mode 100644 index 00000000..b69671ff --- /dev/null +++ b/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java @@ -0,0 +1,96 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.controller.benid; + +import java.util.concurrent.CompletableFuture; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; +import com.iemr.mmu.service.benid.CentralAuthService; +import com.iemr.mmu.utils.response.OutputResponse; + +import io.swagger.v3.oas.annotations.Operation; + +@RestController +@RequestMapping(value = "/centralAuth", headers = "Authorization", produces = "application/json") +public class CentralAuthController { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + @Autowired + private CentralAuthService centralAuthService; + + @Autowired + private BenIdAutoTopUpService benIdAutoTopUpService; + + @Operation(summary = "Automatic BenID top-up and central connection status") + @GetMapping(value = "/status") + public String status() { + OutputResponse response = new OutputResponse(); + try { + response.setResponse(benIdAutoTopUpService.status().toString()); + } catch (Exception e) { + logger.error("Error reading BenID auto top-up status: " + e.getMessage()); + response.setError(e); + } + return response.toString(); + } + + /** + * Called by MMU-UI right after a successful login. Returns immediately; the + * central login and any top-up happen in the background. + */ + @Operation(summary = "Connect this laptop to central with the user's login (one time)") + @PostMapping(value = "/capture", consumes = "application/json") + public String capture(@RequestBody String requestOBJ) { + OutputResponse response = new OutputResponse(); + try { + if (!centralAuthService.isEnabled()) { + response.setResponse("disabled"); + return response.toString(); + } + JSONObject obj = new JSONObject(requestOBJ); + String userName = obj.optString("userName", null); + String password = obj.optString("password", null); + boolean force = obj.optBoolean("reconnect", false); + CompletableFuture.runAsync(() -> { + if (centralAuthService.capture(userName, password, force)) { + benIdAutoTopUpService.triggerNow(); + } + }); + response.setResponse("accepted"); + } catch (Exception e) { + logger.error("Error in central capture: " + e.getMessage()); + response.setError(e); + } + return response.toString(); + } +} diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 3d1f211e..76a9477b 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -37,6 +37,7 @@ import org.springframework.web.bind.annotation.RestController; import com.google.gson.Gson; +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; import com.iemr.mmu.service.dataSyncActivity.DownSyncDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl; @@ -71,6 +72,9 @@ public class StartSyncActivity { @Autowired private DownSyncDataFromServerImpl downSyncDataFromServerImpl; + @Autowired + private BenIdAutoTopUpService benIdAutoTopUpService; + private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; @@ -89,6 +93,8 @@ public String dataSyncToServer(@RequestBody String requestOBJ, serverAuthorization, jwtToken); // if (s != null) response.setResponse(s); + // internet is up right now: top up the BenID pool if it is low + benIdAutoTopUpService.triggerNow(); // else // response.setError(5000, "Error in data sync"); } else { @@ -180,8 +186,11 @@ public String startMasterDownload(@RequestBody String requestOBJ, if (s.equalsIgnoreCase("inProgress")) response.setError(5000, "Download is already in progress from different device, kindly wait to finish this"); - else + else { response.setResponse(s); + // internet is up right now: top up the BenID pool if it is low + benIdAutoTopUpService.triggerNow(); + } } else response.setError(5000, s); } else { diff --git a/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java b/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java new file mode 100644 index 00000000..2e0ec4ce --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java @@ -0,0 +1,322 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.benid; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.locks.ReentrantLock; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import com.iemr.mmu.utils.RestTemplateUtil; + +import jakarta.annotation.PostConstruct; + +/** + * Keeps the local BenID pool filled without anyone clicking "Generate BenID". + * + * Runs only on StopTB laptops (stoptb.enforce.vanid=true and stoptb.van.id>0). + * Pool empty -> bootstrap size; pool below lower limit -> batch size. IDs + * received from central are written to a pending file before the local import, + * so a failed import is retried with the same IDs instead of reserving new ones. + */ +@Service +public class BenIdAutoTopUpService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + /** Shared with the manual "Generate BenID" button so both never run together. */ + public static final ReentrantLock GENERATION_LOCK = new ReentrantLock(); + + private static final String PENDING_FILE_NAME = "benid-pending-import.json"; + private static final long TOKEN_KEEP_ALIVE_MS = 24 * 60 * 60_000L; + + @Autowired + private CentralAuthService centralAuthService; + + @Value("${benGenUrlCentral:}") + private String benGenUrlCentral; + + @Value("${benImportUrlLocal:}") + private String benImportUrlLocal; + + @Value("${benCheckUrlLocal:}") + private String benCheckUrlLocal; + + @Value("${benid.pool.lowerLimit:2000}") + private long lowerLimit; + + @Value("${benid.pool.batchSize:1000}") + private long batchSize; + + @Value("${benid.pool.bootstrapSize:10000}") + private long bootstrapSize; + + @Value("${benid.token.dir:/opt/amrit/data}") + private String dataDir; + + // wait after the 1st, 2nd, 3rd and later failures (5, 15, 30, 60 minutes) + @Value("${benid.autoTopUp.retryDelaysMs:300000,900000,1800000,3600000}") + private long[] retryDelaysMs; + + @Value("${benid.http.connectTimeoutMs:15000}") + private int connectTimeoutMs; + + @Value("${benid.http.readTimeoutMs:60000}") + private int readTimeoutMs; + + private RestTemplate restTemplate; + + private volatile int failureCount; + private volatile long nextAttemptAt; + private volatile long lastPoolCount = -1; + private volatile long lastTopUpAt; + private volatile long lastTopUpCount; + private volatile String lastError; + + @PostConstruct + public void init() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(connectTimeoutMs); + factory.setReadTimeout(readTimeoutMs); + restTemplate = new RestTemplate(factory); + } + + /** Regular check; does nothing on central or non-StopTB deployments. */ + @Scheduled(initialDelayString = "${benid.autoTopUp.initialDelayMs:120000}", fixedDelayString = "${benid.autoTopUp.checkIntervalMs:300000}") + public void scheduledCheck() { + runOnce(false); + } + + /** Run straight away (after a login or a Data Sync), in the background. */ + public void triggerNow() { + if (centralAuthService.isEnabled()) { + CompletableFuture.runAsync(() -> runOnce(true)); + } + } + + private void runOnce(boolean ignoreBackoff) { + if (!centralAuthService.isEnabled()) { + return; + } + if (!ignoreBackoff && System.currentTimeMillis() < nextAttemptAt) { + return; + } + if (!GENERATION_LOCK.tryLock()) { + logger.info("BenID generation already running - skipping this check"); + return; + } + try { + // IDs already received but not saved locally are imported first + if (importPendingIfAny()) { + onSuccess(); + } + + long available = countLocalPool(); + lastPoolCount = available; + long needed = available <= 0 ? bootstrapSize : (available < lowerLimit ? batchSize : 0); + if (needed == 0) { + centralAuthService.keepAliveIfDue(TOKEN_KEEP_ALIVE_MS); + onSuccess(); + return; + } + + String key = centralAuthService.getCentralSessionKey(); + if (key == null) { + lastError = "No central connection - waiting for a user login"; + logger.warn("BenID pool low ({} for vanID {}) but no central token - waiting for a user login", + available, centralAuthService.getVanID()); + return; + } + + int imported = generateAndImport(centralAuthService.getVanID(), needed, key, key, "datasync"); + lastPoolCount = countLocalPool(); + lastTopUpAt = System.currentTimeMillis(); + lastTopUpCount = imported; + logger.info("BenID auto top-up: vanID {} +{} IDs, pool now {}", centralAuthService.getVanID(), imported, + lastPoolCount); + onSuccess(); + } catch (Exception e) { + onFailure(e); + } finally { + GENERATION_LOCK.unlock(); + } + } + + /** + * Asks central for IDs for the van and imports them locally. The central + * response is written to the pending file first, so a failed local import + * does not lose IDs that central has already reserved for this van. + * + * @return number of IDs imported + */ + public int generateAndImport(int vanID, long count, String centralAuthorization, String localAuthorization, + String localToken) throws Exception { + JSONObject request = new JSONObject(); + request.put("vanID", vanID); + request.put("benIDRequired", count); + + HttpEntity centralRequest = RestTemplateUtil.createRequestEntity(request.toString(), + centralAuthorization, "datasync"); + logger.info("Requesting {} BenIDs from central for vanID {}", count, vanID); + long start = System.currentTimeMillis(); + ResponseEntity response = restTemplate.exchange(benGenUrlCentral, HttpMethod.POST, centralRequest, + String.class); + logger.info("Central BenID generation answered in {} ms", System.currentTimeMillis() - start); + JSONObject obj = new JSONObject(response.getBody()); + if (obj.optInt("statusCode") != 200 || !obj.has("data")) { + throw new IllegalStateException("Central BenID generation failed: " + obj.optString("errorMessage")); + } + JSONArray ids = obj.getJSONArray("data"); + for (int k = 0; k < ids.length(); k++) { + ids.getJSONObject(k).put("vanID", vanID); + } + writePending(ids.toString()); + long importStart = System.currentTimeMillis(); + importLocally(ids.toString(), localAuthorization, localToken); + deletePending(); + logger.info("Imported {} BenIDs locally in {} ms", ids.length(), System.currentTimeMillis() - importStart); + return ids.length(); + } + + private boolean importPendingIfAny() throws Exception { + Path pending = pendingPath(); + if (!Files.exists(pending)) { + return false; + } + String payload = new String(Files.readAllBytes(pending), StandardCharsets.UTF_8); + logger.info("Importing BenIDs left from an earlier run"); + try { + importLocally(payload, "datasync", "datasync"); + } catch (Exception e) { + // the earlier run already saved these IDs; nothing left to import + if (e.getMessage() == null || !e.getMessage().toLowerCase().contains("duplicate")) { + throw e; + } + logger.info("Pending BenIDs were already imported earlier"); + } + deletePending(); + return true; + } + + private void importLocally(String payload, String authorization, String token) { + HttpEntity localRequest = RestTemplateUtil.createRequestEntity(payload, authorization, token); + ResponseEntity response = restTemplate.exchange(benImportUrlLocal, HttpMethod.POST, localRequest, + String.class); + JSONObject obj = new JSONObject(response.getBody()); + if (obj.optInt("statusCode") != 200) { + throw new IllegalStateException("Local BenID import failed: " + obj.optString("errorMessage")); + } + } + + public long countLocalPool() { + HttpEntity request = RestTemplateUtil.createRequestEntity(null, "datasync", "datasync"); + ResponseEntity response = restTemplate.exchange(benCheckUrlLocal, HttpMethod.GET, request, + String.class); + JSONObject obj = new JSONObject(response.getBody()); + if (obj.optInt("statusCode") != 200) { + throw new IllegalStateException("Local BenID count failed: " + obj.optString("errorMessage")); + } + Object data = obj.get("data"); + JSONObject dataObj = data instanceof JSONObject ? (JSONObject) data : new JSONObject(data.toString()); + return Long.parseLong(dataObj.get("response").toString()); + } + + private void onSuccess() { + failureCount = 0; + nextAttemptAt = 0; + lastError = null; + } + + private void onFailure(Exception e) { + long delay = retryDelaysMs[Math.min(failureCount, retryDelaysMs.length - 1)]; + failureCount++; + nextAttemptAt = System.currentTimeMillis() + delay; + lastError = e.getMessage(); + logger.warn("BenID auto top-up failed (attempt {}), next try in {} s: {}", failureCount, delay / 1000, + e.getMessage()); + } + + private Path pendingPath() { + return Paths.get(dataDir, PENDING_FILE_NAME); + } + + private void writePending(String payload) { + try { + Files.createDirectories(Paths.get(dataDir)); + Files.write(pendingPath(), payload.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + logger.warn("Could not write pending BenID file: {}", e.getMessage()); + } + } + + private void deletePending() { + try { + Files.deleteIfExists(pendingPath()); + } catch (Exception e) { + logger.warn("Could not delete pending BenID file: {}", e.getMessage()); + } + } + + // ---------------------------------------------------------------- status + + public JSONObject status() { + JSONObject s = new JSONObject(); + s.put("enabled", centralAuthService.isEnabled()); + if (!centralAuthService.isEnabled()) { + return s; + } + s.put("vanID", centralAuthService.getVanID()); + s.put("centralConnection", centralAuthService.getState().name()); + s.put("connectedUser", centralAuthService.getConnectedUser()); + try { + lastPoolCount = countLocalPool(); + } catch (Exception e) { + logger.debug("Live BenID count not available: {}", e.getMessage()); + } + s.put("poolCount", lastPoolCount); + s.put("lowerLimit", lowerLimit); + s.put("lastTopUpAt", lastTopUpAt); + s.put("lastTopUpCount", lastTopUpCount); + s.put("pendingImport", Files.exists(pendingPath())); + String error = lastError != null ? lastError : centralAuthService.getLastError(); + if (error != null) { + s.put("lastError", error); + } + return s; + } +} diff --git a/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java b/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java new file mode 100644 index 00000000..2af2d34f --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java @@ -0,0 +1,377 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.benid; + +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.MessageDigest; +import java.security.SecureRandom; +import java.util.Base64; + +import javax.crypto.Cipher; +import javax.crypto.spec.GCMParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.client.SimpleClientHttpRequestFactory; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; + +import jakarta.annotation.PostConstruct; + +/** + * Keeps this laptop connected to the central server without storing any + * central password. + * + * The operator's normal laptop login is passed here once; it is used for a + * single central login as an API client, which returns a refresh token. Only + * that refresh token is kept (AES-GCM encrypted with a key tied to this + * machine). Each time a central session is needed the refresh token is + * exchanged for a new JWT + refresh token, and the JWT for a central session + * key. + */ +@Service +public class CentralAuthService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + public enum State { + DISABLED, NOT_CONNECTED, CONNECTED, RECONNECT_NEEDED, PASSWORD_MISMATCH + } + + // Central refresh tokens are only issued to API (mobile) clients. + private static final String API_CLIENT_USER_AGENT = "okhttp/4.12.0"; + private static final String TOKEN_FILE_NAME = "central-refresh-token"; + + @Value("${stoptb.enforce.vanid:false}") + private boolean enforceVanID; + + @Value("${stoptb.van.id:0}") + private int vanID; + + @Value("${centralLoginUrl:}") + private String centralLoginUrl; + + @Value("${centralRefreshTokenUrl:}") + private String centralRefreshTokenUrl; + + @Value("${centralSessionUrl:}") + private String centralSessionUrl; + + @Value("${benid.token.dir:/opt/amrit/data}") + private String tokenDir; + + @Value("${benid.token.secret:amrit-mmu-benid}") + private String tokenSecret; + + @Value("${benid.http.connectTimeoutMs:15000}") + private int connectTimeoutMs; + + @Value("${benid.http.readTimeoutMs:60000}") + private int readTimeoutMs; + + private RestTemplate restTemplate; + + // in-memory copy; the file is only a durable backup of this value + private volatile String refreshToken; + private volatile State state = State.NOT_CONNECTED; + private volatile String lastError; + private volatile long lastRefreshAt; + private volatile String connectedUser; + private volatile String rejectedUser; + + @PostConstruct + public void init() { + SimpleClientHttpRequestFactory factory = new SimpleClientHttpRequestFactory(); + factory.setConnectTimeout(connectTimeoutMs); + factory.setReadTimeout(readTimeoutMs); + restTemplate = new RestTemplate(factory); + + if (!isEnabled()) { + state = State.DISABLED; + logger.info("BenID auto top-up disabled (stoptb.enforce.vanid={}, stoptb.van.id={})", enforceVanID, + vanID); + return; + } + refreshToken = readTokenFile(); + state = refreshToken != null ? State.CONNECTED : State.NOT_CONNECTED; + logger.info("BenID auto top-up enabled for vanID {}; central token {}", vanID, + refreshToken != null ? "loaded" : "not present yet"); + } + + public boolean isEnabled() { + return enforceVanID && vanID > 0 && !centralLoginUrl.isEmpty(); + } + + public int getVanID() { + return vanID; + } + + public boolean hasToken() { + return refreshToken != null; + } + + public State getState() { + return state; + } + + public String getLastError() { + return lastError; + } + + public long getLastRefreshAt() { + return lastRefreshAt; + } + + public String getConnectedUser() { + return connectedUser; + } + + /** + * One central login with the operator's credentials (password already + * encrypted the way MMU-UI sends it to Common-API). Skipped when a working + * token already exists, so central is not hit on every laptop login. A + * rejected password is never retried, so the central account cannot get + * locked by this. + * + * @param force true when the user explicitly reconnects (e.g. Data Sync + * login), even if a token exists + * @return true if a new token was saved + */ + public synchronized boolean capture(String userName, String encryptedPassword, boolean force) { + if (!isEnabled() || userName == null || encryptedPassword == null) { + return false; + } + if (!force && refreshToken != null && state == State.CONNECTED) { + return false; + } + // a laptop password central already rejected is not sent again (account lock risk); + // only a Data Sync login (force, real central password) tries again + if (!force && userName.trim().equalsIgnoreCase(rejectedUser)) { + logger.info("Central login skipped for user {}: password was rejected before", userName); + return false; + } + try { + JSONObject body = new JSONObject(); + body.put("userName", userName.trim()); + body.put("password", encryptedPassword); + body.put("doLogout", false); + + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set(HttpHeaders.USER_AGENT, API_CLIENT_USER_AGENT); + + ResponseEntity response = restTemplate.exchange(centralLoginUrl, HttpMethod.POST, + new HttpEntity<>(body.toString(), headers), String.class); + JSONObject obj = new JSONObject(response.getBody()); + if (obj.optInt("statusCode") == 200 && obj.has("data") + && obj.getJSONObject("data").has("refreshToken")) { + saveToken(obj.getJSONObject("data").getString("refreshToken")); + connectedUser = userName.trim(); + rejectedUser = null; + state = State.CONNECTED; + lastError = null; + logger.info("Central token captured for user {}", connectedUser); + return true; + } + // wrong username/password or account problem: do not retry with these credentials + lastError = obj.optString("errorMessage", "Central login rejected"); + rejectedUser = userName.trim(); + state = State.PASSWORD_MISMATCH; + logger.warn("Central login rejected for user {}: {}", userName, lastError); + } catch (Exception e) { + // network problem: nothing saved, next login tries again + lastError = "Central server not reachable: " + e.getMessage(); + logger.warn("Central login not possible right now: {}", e.getMessage()); + } + return false; + } + + /** + * Returns a fresh central session key (refreshToken -> JWT -> session key), + * or null if this laptop has no usable token. Throws on network errors so + * callers can retry later. + */ + public synchronized String getCentralSessionKey() throws Exception { + if (!isEnabled() || refreshToken == null) { + return null; + } + String jwt; + long start = System.currentTimeMillis(); + try { + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + JSONObject body = new JSONObject(); + body.put("refreshToken", refreshToken); + ResponseEntity response = restTemplate.exchange(centralRefreshTokenUrl, HttpMethod.POST, + new HttpEntity<>(body.toString(), headers), String.class); + JSONObject obj = new JSONObject(response.getBody()); + jwt = obj.getString("jwtToken"); + if (obj.has("refreshToken")) { + saveToken(obj.getString("refreshToken")); + } + lastRefreshAt = System.currentTimeMillis(); + logger.info("Central token refreshed in {} ms", lastRefreshAt - start); + } catch (HttpStatusCodeException e) { + if (e.getStatusCode().value() == 401) { + // expired, revoked or user deactivated on central + logger.warn("Central refresh token rejected ({}); a new login is needed", + e.getResponseBodyAsString()); + deleteToken(); + state = State.RECONNECT_NEEDED; + lastError = "Central connection expired, please log in again"; + return null; + } + throw e; + } + + // JWT -> central session key; Common-API creates a new Redis session for it + long sessionStart = System.currentTimeMillis(); + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + headers.set("Jwttoken", jwt); + ResponseEntity response = restTemplate.exchange(centralSessionUrl, HttpMethod.POST, + new HttpEntity<>("{}", headers), String.class); + JSONObject obj = new JSONObject(response.getBody()); + if (obj.optInt("statusCode") != 200 || !obj.has("data")) { + throw new IllegalStateException("Central session not created: " + obj.optString("errorMessage")); + } + Object data = obj.get("data"); + JSONObject dataObj = data instanceof JSONObject ? (JSONObject) data : new JSONObject(data.toString()); + state = State.CONNECTED; + lastError = null; + logger.info("Central session created in {} ms", System.currentTimeMillis() - sessionStart); + return dataObj.getString("key"); + } + + /** Keeps the 90-day token alive even when no IDs are needed. */ + public void keepAliveIfDue(long maxAgeMs) { + if (refreshToken != null && System.currentTimeMillis() - lastRefreshAt > maxAgeMs) { + try { + getCentralSessionKey(); + } catch (Exception e) { + logger.debug("Token keep-alive skipped: {}", e.getMessage()); + } + } + } + + // ---------------------------------------------------------------- token file + + private Path tokenPath() { + return Paths.get(tokenDir, TOKEN_FILE_NAME); + } + + private void saveToken(String token) { + refreshToken = token; + try { + Path dir = Paths.get(tokenDir); + Files.createDirectories(dir); + setPermissions(dir, "rwx------"); + Path tmp = dir.resolve(TOKEN_FILE_NAME + ".tmp"); + Files.write(tmp, encrypt(token).getBytes(StandardCharsets.UTF_8)); + setPermissions(tmp, "rw-------"); + try { + Files.move(tmp, tokenPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (AtomicMoveNotSupportedException e) { + Files.move(tmp, tokenPath(), StandardCopyOption.REPLACE_EXISTING); + } + } catch (Exception e) { + // keep working from memory; the token is captured again after a restart + logger.warn("Could not save central token to {} (kept in memory only): {}", tokenDir, e.getMessage()); + } + } + + private String readTokenFile() { + try { + Path path = tokenPath(); + if (Files.exists(path)) { + return decrypt(new String(Files.readAllBytes(path), StandardCharsets.UTF_8).trim()); + } + } catch (Exception e) { + logger.warn("Stored central token could not be read, a new login is needed: {}", e.getMessage()); + } + return null; + } + + private void deleteToken() { + refreshToken = null; + try { + Files.deleteIfExists(tokenPath()); + } catch (Exception e) { + logger.warn("Could not delete central token file: {}", e.getMessage()); + } + } + + private static void setPermissions(Path path, String perms) { + try { + Files.setPosixFilePermissions(path, PosixFilePermissions.fromString(perms)); + } catch (Exception e) { + // non-POSIX file system; nothing more we can do + } + } + + private SecretKeySpec machineKey() throws Exception { + String machineId; + try { + machineId = new String(Files.readAllBytes(Paths.get("/etc/machine-id")), StandardCharsets.UTF_8).trim(); + } catch (Exception e) { + machineId = InetAddress.getLocalHost().getHostName(); + } + byte[] key = MessageDigest.getInstance("SHA-256") + .digest((machineId + ":" + tokenSecret).getBytes(StandardCharsets.UTF_8)); + return new SecretKeySpec(key, "AES"); + } + + private String encrypt(String plain) throws Exception { + byte[] iv = new byte[12]; + new SecureRandom().nextBytes(iv); + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.ENCRYPT_MODE, machineKey(), new GCMParameterSpec(128, iv)); + byte[] enc = cipher.doFinal(plain.getBytes(StandardCharsets.UTF_8)); + byte[] out = new byte[iv.length + enc.length]; + System.arraycopy(iv, 0, out, 0, iv.length); + System.arraycopy(enc, 0, out, iv.length, enc.length); + return Base64.getEncoder().encodeToString(out); + } + + private String decrypt(String encoded) throws Exception { + byte[] in = Base64.getDecoder().decode(encoded); + Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding"); + cipher.init(Cipher.DECRYPT_MODE, machineKey(), new GCMParameterSpec(128, in, 0, 12)); + return new String(cipher.doFinal(in, 12, in.length - 12), StandardCharsets.UTF_8); + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java index 6a6d12bc..3ad4eaa8 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java @@ -30,6 +30,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.json.JSONObject; import org.slf4j.Logger; @@ -45,6 +46,7 @@ import com.google.gson.Gson; import com.iemr.mmu.data.syncActivity_syncLayer.MasterDownloadDataDigester; +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; import com.iemr.mmu.data.syncActivity_syncLayer.SyncDownloadMaster; import com.iemr.mmu.data.syncActivity_syncLayer.TempVan; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncDownloadMasterRepo; @@ -326,6 +328,10 @@ public Map getDownloadStatus() { public int callCentralAPIToGenerateBenIDAndimportToLocal(String requestOBJ, String Authorization, String ServerAuthorization, String token) throws Exception { int i = 0, i1 = 0; + // never run together with the automatic top-up, so the same IDs are not requested twice + if (!BenIdAutoTopUpService.GENERATION_LOCK.tryLock(2, TimeUnit.MINUTES)) { + throw new Exception("BenID generation is already running, please try again in a few minutes"); + } try{ JSONObject originalRequest = new JSONObject(requestOBJ); BigInteger vanID = null; @@ -378,6 +384,8 @@ public int callCentralAPIToGenerateBenIDAndimportToLocal(String requestOBJ, Stri } catch (Exception e) { logger.error("Error while generating catch UNIQUE_ID at central server: " + e.getMessage()); throw new Exception("Error while generating catch UNIQUE_ID at central server: " + e.getMessage()); + } finally { + BenIdAutoTopUpService.GENERATION_LOCK.unlock(); } return i; } diff --git a/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java b/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java new file mode 100644 index 00000000..e075e387 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java @@ -0,0 +1,225 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.benid; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.content; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.test.web.client.ExpectedCount; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; + +class BenIdAutoTopUpServiceTest { + + private static final String GENERATE = "http://central/common-api/beneficiary/generateBeneficiaryIDs"; + private static final String IMPORT = "http://local/id/saveGeneratedBenIDToLocalServer"; + private static final String COUNT = "http://local/id/checkAvailablBenIDLocalServer"; + + @TempDir + Path dataDir; + + private BenIdAutoTopUpService service; + private CentralAuthService centralAuth; + private MockRestServiceServer server; + + @BeforeEach + void setUp() throws Exception { + centralAuth = mock(CentralAuthService.class); + when(centralAuth.isEnabled()).thenReturn(true); + when(centralAuth.getVanID()).thenReturn(403); + when(centralAuth.getState()).thenReturn(CentralAuthService.State.CONNECTED); + when(centralAuth.getCentralSessionKey()).thenReturn("SESSION-KEY"); + + service = new BenIdAutoTopUpService(); + ReflectionTestUtils.setField(service, "centralAuthService", centralAuth); + ReflectionTestUtils.setField(service, "benGenUrlCentral", GENERATE); + ReflectionTestUtils.setField(service, "benImportUrlLocal", IMPORT); + ReflectionTestUtils.setField(service, "benCheckUrlLocal", COUNT); + ReflectionTestUtils.setField(service, "lowerLimit", 2000L); + ReflectionTestUtils.setField(service, "batchSize", 1000L); + ReflectionTestUtils.setField(service, "bootstrapSize", 10000L); + ReflectionTestUtils.setField(service, "dataDir", dataDir.toString()); + ReflectionTestUtils.setField(service, "retryDelaysMs", new long[] { 0L }); + ReflectionTestUtils.setField(service, "connectTimeoutMs", 1000); + ReflectionTestUtils.setField(service, "readTimeoutMs", 1000); + service.init(); + server = MockRestServiceServer.bindTo((RestTemplate) ReflectionTestUtils.getField(service, "restTemplate")) + .build(); + } + + @AfterEach + void releaseLock() { + while (BenIdAutoTopUpService.GENERATION_LOCK.isHeldByCurrentThread()) { + BenIdAutoTopUpService.GENERATION_LOCK.unlock(); + } + } + + private void expectCount(long n) { + server.expect(requestTo(COUNT)) + .andRespond(withSuccess("{\"statusCode\":200,\"data\":{\"response\":\"" + n + "\"}}", + MediaType.APPLICATION_JSON)); + } + + private static String ids(int n) { + StringBuilder sb = new StringBuilder("["); + for (int i = 1; i <= n; i++) { + sb.append(i > 1 ? "," : "").append("{\"benRegId\":").append(5000 + i) + .append(",\"beneficiaryId\":").append(900000000000L + i).append("}"); + } + return sb.append("]").toString(); + } + + private void expectGenerate(long requested, int returned) { + server.expect(requestTo(GENERATE)).andExpect(header("Authorization", "SESSION-KEY")) + .andExpect(content().string(Matchers.containsString("\"benIDRequired\":" + requested))) + .andExpect(content().string(Matchers.containsString("\"vanID\":403"))) + .andRespond(withSuccess("{\"statusCode\":200,\"data\":" + ids(returned) + "}", + MediaType.APPLICATION_JSON)); + } + + private void expectImportOk() { + server.expect(requestTo(IMPORT)).andExpect(content().string(Matchers.containsString("\"vanID\":403"))) + .andRespond(withSuccess("{\"statusCode\":200,\"data\":{\"response\":\"ok\"}}", + MediaType.APPLICATION_JSON)); + } + + private void run() { + ReflectionTestUtils.invokeMethod(service, "runOnce", true); + } + + private Path pending() { + return dataDir.resolve("benid-pending-import.json"); + } + + @Test + void doesNothingWhenDisabled() throws Exception { + when(centralAuth.isEnabled()).thenReturn(false); + server.expect(ExpectedCount.never(), requestTo(COUNT)); + run(); + server.verify(); + verify(centralAuth, never()).getCentralSessionKey(); + } + + @Test + void emptyPoolGetsBootstrapSize() { + expectCount(0); + expectGenerate(10000, 3); + expectImportOk(); + expectCount(3); + run(); + server.verify(); + assertFalse(Files.exists(pending()), "pending file removed after a successful import"); + } + + @Test + void lowPoolGetsBatchSize() { + expectCount(1999); + expectGenerate(1000, 2); + expectImportOk(); + expectCount(2001); + run(); + server.verify(); + } + + @Test + void healthyPoolDoesNotCallCentral() throws Exception { + expectCount(2000); + server.expect(ExpectedCount.never(), requestTo(GENERATE)); + run(); + server.verify(); + verify(centralAuth, never()).getCentralSessionKey(); + } + + @Test + void noCentralTokenWaitsForLogin() throws Exception { + when(centralAuth.getCentralSessionKey()).thenReturn(null); + expectCount(0); + server.expect(ExpectedCount.never(), requestTo(GENERATE)); + run(); + server.verify(); + assertTrue(((String) ReflectionTestUtils.getField(service, "lastError")).contains("waiting for a user login")); + } + + @Test + void centralErrorLeavesNothingBehind() { + expectCount(0); + server.expect(requestTo(GENERATE)).andRespond(withStatus(HttpStatus.GATEWAY_TIMEOUT)); + run(); + server.verify(); + assertFalse(Files.exists(pending())); + assertTrue(ReflectionTestUtils.getField(service, "lastError") != null); + } + + @Test + void failedLocalImportKeepsIdsAndImportsThemNextRun() throws Exception { + expectCount(0); + expectGenerate(10000, 3); + server.expect(requestTo(IMPORT)).andRespond(withStatus(HttpStatus.INTERNAL_SERVER_ERROR)); + // next run: pending IDs are imported first, then the pool is healthy + expectImportOk(); + expectCount(2500); + run(); + assertTrue(Files.exists(pending()), "IDs received from central must be kept"); + assertEquals(3, Files.readString(pending()).split("benRegId").length - 1); + + run(); + server.verify(); + assertFalse(Files.exists(pending())); + } + + @Test + void skipsWhileManualGenerationRuns() { + BenIdAutoTopUpService.GENERATION_LOCK.lock(); + try { + Thread other = new Thread(this::run); + server.expect(ExpectedCount.never(), requestTo(COUNT)); + other.start(); + other.join(5000); + server.verify(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + BenIdAutoTopUpService.GENERATION_LOCK.unlock(); + } + } +} diff --git a/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java b/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java new file mode 100644 index 00000000..744dd16e --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java @@ -0,0 +1,245 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.benid; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.content; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.method; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.util.ReflectionTestUtils; +import org.springframework.test.web.client.ExpectedCount; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; + +class CentralAuthServiceTest { + + private static final String LOGIN = "http://central/common-api/user/userAuthenticate"; + private static final String REFRESH = "http://central/common-api/user/refreshToken"; + private static final String SESSION = "http://central/common-api/user/getLoginResponse"; + + @TempDir + Path tokenDir; + + private CentralAuthService service; + private MockRestServiceServer central; + + @BeforeEach + void setUp() { + service = newService(true, 403); + } + + private CentralAuthService newService(boolean enforce, int vanID) { + CentralAuthService s = new CentralAuthService(); + ReflectionTestUtils.setField(s, "enforceVanID", enforce); + ReflectionTestUtils.setField(s, "vanID", vanID); + ReflectionTestUtils.setField(s, "centralLoginUrl", LOGIN); + ReflectionTestUtils.setField(s, "centralRefreshTokenUrl", REFRESH); + ReflectionTestUtils.setField(s, "centralSessionUrl", SESSION); + ReflectionTestUtils.setField(s, "tokenDir", tokenDir.toString()); + ReflectionTestUtils.setField(s, "tokenSecret", "test-secret"); + ReflectionTestUtils.setField(s, "connectTimeoutMs", 1000); + ReflectionTestUtils.setField(s, "readTimeoutMs", 1000); + s.init(); + RestTemplate rt = (RestTemplate) ReflectionTestUtils.getField(s, "restTemplate"); + central = MockRestServiceServer.bindTo(rt).build(); + return s; + } + + private void expectLoginOk(String refreshToken) { + central.expect(requestTo(LOGIN)).andExpect(method(HttpMethod.POST)) + .andExpect(header("User-Agent", "okhttp/4.12.0")) + .andExpect(content().string(org.hamcrest.Matchers.containsString("\"doLogout\":false"))) + .andRespond(withSuccess("{\"statusCode\":200,\"data\":{\"refreshToken\":\"" + refreshToken + + "\",\"key\":\"k\"}}", MediaType.APPLICATION_JSON)); + } + + private void expectLoginRejected() { + central.expect(requestTo(LOGIN)).andRespond(withSuccess( + "{\"statusCode\":5002,\"errorMessage\":\"Invalid username or password\"}", MediaType.APPLICATION_JSON)); + } + + @Test + void disabledWithoutVanId() { + CentralAuthService off = newService(true, 0); + assertFalse(off.isEnabled()); + assertEquals(CentralAuthService.State.DISABLED, off.getState()); + assertFalse(off.capture("stoptb1", "pw", true)); + central.verify(); // nothing sent to central + } + + @Test + void disabledWhenEnforceFlagOff() { + CentralAuthService off = newService(false, 403); + assertFalse(off.isEnabled()); + assertFalse(off.capture("stoptb1", "pw", true)); + central.verify(); + } + + @Test + void captureStoresOnlyEncryptedTokenWithOwnerOnlyPermissions() throws Exception { + expectLoginOk("RT-secret-123"); + + assertTrue(service.capture("stoptb1", "encrypted-pw", false)); + + Path file = tokenDir.resolve("central-refresh-token"); + String stored = Files.readString(file, StandardCharsets.UTF_8); + assertFalse(stored.contains("RT-secret-123"), "token must not be stored in plain text"); + assertFalse(stored.contains("encrypted-pw"), "password must never be stored"); + assertEquals("rw-------", PosixFilePermissions.toString(Files.getPosixFilePermissions(file))); + assertEquals(CentralAuthService.State.CONNECTED, service.getState()); + central.verify(); + } + + @Test + void tokenIsReadBackAfterRestart() { + expectLoginOk("RT-1"); + service.capture("stoptb1", "pw", false); + + CentralAuthService restarted = newService(true, 403); + assertTrue(restarted.hasToken()); + assertEquals(CentralAuthService.State.CONNECTED, restarted.getState()); + } + + @Test + void corruptTokenFileMeansNewLoginNeeded() throws Exception { + Files.writeString(tokenDir.resolve("central-refresh-token"), "not-a-valid-token"); + CentralAuthService s = newService(true, 403); + assertFalse(s.hasToken()); + assertEquals(CentralAuthService.State.NOT_CONNECTED, s.getState()); + } + + @Test + void secondLoginIsSkippedWhileTokenWorks() { + expectLoginOk("RT-1"); + service.capture("stoptb1", "pw", false); + + assertFalse(service.capture("stoptb1", "pw", false)); + central.verify(); // only one central login + } + + @Test + void rejectedPasswordIsNeverSentAgainWithoutReconnect() { + expectLoginRejected(); + + assertFalse(service.capture("stoptb1", "wrong", false)); + assertEquals(CentralAuthService.State.PASSWORD_MISMATCH, service.getState()); + // next laptop login with the same user must not reach central (account lock risk) + assertFalse(service.capture("stoptb1", "wrong", false)); + central.verify(); + } + + @Test + void dataSyncReconnectRetriesAfterRejection() { + expectLoginRejected(); + expectLoginOk("RT-2"); + + service.capture("stoptb1", "wrong", false); + assertTrue(service.capture("stoptb1", "central-pw", true)); + assertEquals(CentralAuthService.State.CONNECTED, service.getState()); + central.verify(); + } + + @Test + void centralDownIsNotTreatedAsWrongPassword() { + central.expect(requestTo(LOGIN)).andRespond(withStatus(HttpStatus.BAD_GATEWAY)); + expectLoginOk("RT-3"); + + assertFalse(service.capture("stoptb1", "pw", false)); + // network problem: the next login tries again + assertTrue(service.capture("stoptb1", "pw", false)); + central.verify(); + } + + @Test + void sessionKeyFromRefreshTokenAndTokenRotation() throws Exception { + expectLoginOk("RT-old"); + central.expect(requestTo(REFRESH)) + .andExpect(content().string(org.hamcrest.Matchers.containsString("RT-old"))) + .andRespond(withSuccess("{\"jwtToken\":\"JWT-1\",\"refreshToken\":\"RT-new\"}", + MediaType.APPLICATION_JSON)); + central.expect(requestTo(SESSION)).andExpect(header("Jwttoken", "JWT-1")) + .andRespond(withSuccess("{\"statusCode\":200,\"data\":{\"key\":\"SESSION-KEY\"}}", + MediaType.APPLICATION_JSON)); + // the next refresh must use the rotated token + central.expect(requestTo(REFRESH)) + .andExpect(content().string(org.hamcrest.Matchers.containsString("RT-new"))) + .andRespond(withSuccess("{\"jwtToken\":\"JWT-2\"}", MediaType.APPLICATION_JSON)); + central.expect(requestTo(SESSION)).andRespond(withSuccess( + "{\"statusCode\":200,\"data\":\"{\\\"key\\\":\\\"KEY-2\\\"}\"}", MediaType.APPLICATION_JSON)); + + service.capture("stoptb1", "pw", false); + String fileBefore = Files.readString(tokenDir.resolve("central-refresh-token")); + + assertEquals("SESSION-KEY", service.getCentralSessionKey()); + assertFalse(fileBefore.equals(Files.readString(tokenDir.resolve("central-refresh-token"))), + "rotated refresh token must be saved"); + assertEquals("KEY-2", service.getCentralSessionKey()); + central.verify(); + } + + @Test + void revokedTokenIsDeletedAndReconnectNeeded() { + expectLoginOk("RT-1"); + central.expect(requestTo(REFRESH)).andRespond(withStatus(HttpStatus.UNAUTHORIZED)); + + service.capture("stoptb1", "pw", false); + + assertNull(assertDoesNotThrowValue()); + assertFalse(service.hasToken()); + assertFalse(Files.exists(tokenDir.resolve("central-refresh-token"))); + assertEquals(CentralAuthService.State.RECONNECT_NEEDED, service.getState()); + central.verify(); + } + + @Test + void noTokenMeansNoCentralCall() throws Exception { + central.expect(ExpectedCount.never(), requestTo(REFRESH)); + assertNull(service.getCentralSessionKey()); + central.verify(); + } + + private String assertDoesNotThrowValue() { + try { + return service.getCentralSessionKey(); + } catch (Exception e) { + throw new AssertionError("revoked token must not throw", e); + } + } +} From 738ed2f6232f87c4a719a739d57860771a92473f Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Wed, 14 Oct 2026 16:07:15 +0530 Subject: [PATCH 78/81] fix(STOP-457): harden automatic BenID download for many laptops - Ask central for at most 1000 IDs per request (benid.pool.maxPerRequest): a 10000-ID request can outlast the central gateway timeout, and every retry after a lost reply reserved another 10000 unused IDs. - Remember a central-rejected laptop password across restarts (marker file with the user name only), so daily restarts cannot slowly lock the central account; a successful Data Sync reconnect clears it. - Write the pending-import file atomically (temp + move); a pending file that still fails after 3 tries is set aside so top-ups never get stuck. - Spread retry waits by +/-20% so laptops do not all hit central at once after an outage. - Run background work on the feature's own single thread instead of the shared ForkJoin pool. Tests: 26 unit tests pass; checked in the running app (2500 IDs arrived as 1000+1000+500; rejected login not resent after restart). Co-Authored-By: Claude Opus 5.5 --- .../benid/CentralAuthController.java | 4 +- .../service/benid/BenIdAutoTopUpService.java | 68 ++++++++++++++++--- .../mmu/service/benid/CentralAuthService.java | 39 +++++++++++ .../benid/BenIdAutoTopUpServiceTest.java | 51 +++++++++++++- .../service/benid/CentralAuthServiceTest.java | 31 +++++++++ 5 files changed, 179 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java b/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java index b69671ff..71533a5d 100644 --- a/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java +++ b/src/main/java/com/iemr/mmu/controller/benid/CentralAuthController.java @@ -21,8 +21,6 @@ */ package com.iemr.mmu.controller.benid; -import java.util.concurrent.CompletableFuture; - import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,7 +79,7 @@ public String capture(@RequestBody String requestOBJ) { String userName = obj.optString("userName", null); String password = obj.optString("password", null); boolean force = obj.optBoolean("reconnect", false); - CompletableFuture.runAsync(() -> { + benIdAutoTopUpService.runInBackground(() -> { if (centralAuthService.capture(userName, password, force)) { benIdAutoTopUpService.triggerNow(); } diff --git a/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java b/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java index 2e0ec4ce..56b78f16 100644 --- a/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java +++ b/src/main/java/com/iemr/mmu/service/benid/BenIdAutoTopUpService.java @@ -25,7 +25,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.concurrent.CompletableFuture; +import java.nio.file.StandardCopyOption; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.locks.ReentrantLock; import org.json.JSONArray; @@ -45,6 +48,7 @@ import com.iemr.mmu.utils.RestTemplateUtil; import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; /** * Keeps the local BenID pool filled without anyone clicking "Generate BenID". @@ -86,6 +90,11 @@ public class BenIdAutoTopUpService { @Value("${benid.pool.bootstrapSize:10000}") private long bootstrapSize; + // central is asked for at most this many IDs per request: a big request can outlast the + // central gateway timeout, and a reply lost after central created the IDs wastes them + @Value("${benid.pool.maxPerRequest:1000}") + private long maxPerRequest; + @Value("${benid.token.dir:/opt/amrit/data}") private String dataDir; @@ -107,6 +116,16 @@ public class BenIdAutoTopUpService { private volatile long lastTopUpAt; private volatile long lastTopUpCount; private volatile String lastError; + private volatile int pendingImportFailures; + + // one background thread for this feature, so network waits never hold shared pool threads + private final ExecutorService background = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "benid-auto-topup"); + t.setDaemon(true); + return t; + }); + + private static final int MAX_PENDING_IMPORT_FAILURES = 3; @PostConstruct public void init() { @@ -116,6 +135,16 @@ public void init() { restTemplate = new RestTemplate(factory); } + @PreDestroy + public void shutdown() { + background.shutdownNow(); + } + + /** Runs a task on this feature's own background thread. */ + public void runInBackground(Runnable task) { + background.submit(task); + } + /** Regular check; does nothing on central or non-StopTB deployments. */ @Scheduled(initialDelayString = "${benid.autoTopUp.initialDelayMs:120000}", fixedDelayString = "${benid.autoTopUp.checkIntervalMs:300000}") public void scheduledCheck() { @@ -125,7 +154,7 @@ public void scheduledCheck() { /** Run straight away (after a login or a Data Sync), in the background. */ public void triggerNow() { if (centralAuthService.isEnabled()) { - CompletableFuture.runAsync(() -> runOnce(true)); + background.submit(() -> runOnce(true)); } } @@ -163,7 +192,12 @@ private void runOnce(boolean ignoreBackoff) { return; } - int imported = generateAndImport(centralAuthService.getVanID(), needed, key, key, "datasync"); + int imported = 0; + for (long remaining = needed; remaining > 0;) { + long chunk = Math.min(remaining, Math.max(1, maxPerRequest)); + imported += generateAndImport(centralAuthService.getVanID(), chunk, key, key, "datasync"); + remaining -= chunk; + } lastPoolCount = countLocalPool(); lastTopUpAt = System.currentTimeMillis(); lastTopUpCount = imported; @@ -218,17 +252,24 @@ private boolean importPendingIfAny() throws Exception { if (!Files.exists(pending)) { return false; } - String payload = new String(Files.readAllBytes(pending), StandardCharsets.UTF_8); logger.info("Importing BenIDs left from an earlier run"); try { + String payload = new String(Files.readAllBytes(pending), StandardCharsets.UTF_8); importLocally(payload, "datasync", "datasync"); } catch (Exception e) { - // the earlier run already saved these IDs; nothing left to import - if (e.getMessage() == null || !e.getMessage().toLowerCase().contains("duplicate")) { + pendingImportFailures++; + if (pendingImportFailures < MAX_PENDING_IMPORT_FAILURES) { throw e; } - logger.info("Pending BenIDs were already imported earlier"); + // a file that keeps failing (e.g. half-written at a power cut) must not stop top-ups + Path setAside = Paths.get(dataDir, "benid-pending-import.failed-" + System.currentTimeMillis() + ".json"); + Files.move(pending, setAside, StandardCopyOption.REPLACE_EXISTING); + pendingImportFailures = 0; + logger.warn("Pending BenID file could not be imported {} times; moved to {} and continuing: {}", + MAX_PENDING_IMPORT_FAILURES, setAside, e.getMessage()); + return false; } + pendingImportFailures = 0; deletePending(); return true; } @@ -263,7 +304,9 @@ private void onSuccess() { } private void onFailure(Exception e) { - long delay = retryDelaysMs[Math.min(failureCount, retryDelaysMs.length - 1)]; + long base = retryDelaysMs[Math.min(failureCount, retryDelaysMs.length - 1)]; + // +/-20% so that many laptops do not all retry at the same moment after an outage + long delay = base <= 0 ? 0 : (long) (base * ThreadLocalRandom.current().nextDouble(0.8, 1.2)); failureCount++; nextAttemptAt = System.currentTimeMillis() + delay; lastError = e.getMessage(); @@ -278,7 +321,14 @@ private Path pendingPath() { private void writePending(String payload) { try { Files.createDirectories(Paths.get(dataDir)); - Files.write(pendingPath(), payload.getBytes(StandardCharsets.UTF_8)); + // write a temp file and swap it in, so a power cut never leaves a half-written file + Path tmp = Paths.get(dataDir, PENDING_FILE_NAME + ".tmp"); + Files.write(tmp, payload.getBytes(StandardCharsets.UTF_8)); + try { + Files.move(tmp, pendingPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (java.nio.file.AtomicMoveNotSupportedException e) { + Files.move(tmp, pendingPath(), StandardCopyOption.REPLACE_EXISTING); + } } catch (Exception e) { logger.warn("Could not write pending BenID file: {}", e.getMessage()); } diff --git a/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java b/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java index 2af2d34f..d065eb19 100644 --- a/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java +++ b/src/main/java/com/iemr/mmu/service/benid/CentralAuthService.java @@ -76,6 +76,9 @@ public enum State { // Central refresh tokens are only issued to API (mobile) clients. private static final String API_CLIENT_USER_AGENT = "okhttp/4.12.0"; private static final String TOKEN_FILE_NAME = "central-refresh-token"; + // user name only (no password): a laptop password central rejected is not sent again, + // also after a restart, so the central account cannot get locked by repeated attempts + private static final String REJECTED_FILE_NAME = "central-login-rejected"; @Value("${stoptb.enforce.vanid:false}") private boolean enforceVanID; @@ -128,6 +131,7 @@ public void init() { return; } refreshToken = readTokenFile(); + rejectedUser = readRejectedUser(); state = refreshToken != null ? State.CONNECTED : State.NOT_CONNECTED; logger.info("BenID auto top-up enabled for vanID {}; central token {}", vanID, refreshToken != null ? "loaded" : "not present yet"); @@ -203,6 +207,7 @@ public synchronized boolean capture(String userName, String encryptedPassword, b saveToken(obj.getJSONObject("data").getString("refreshToken")); connectedUser = userName.trim(); rejectedUser = null; + deleteRejectedUser(); state = State.CONNECTED; lastError = null; logger.info("Central token captured for user {}", connectedUser); @@ -211,6 +216,7 @@ public synchronized boolean capture(String userName, String encryptedPassword, b // wrong username/password or account problem: do not retry with these credentials lastError = obj.optString("errorMessage", "Central login rejected"); rejectedUser = userName.trim(); + saveRejectedUser(rejectedUser); state = State.PASSWORD_MISMATCH; logger.warn("Central login rejected for user {}: {}", userName, lastError); } catch (Exception e) { @@ -336,6 +342,39 @@ private void deleteToken() { } } + private String readRejectedUser() { + try { + Path path = Paths.get(tokenDir, REJECTED_FILE_NAME); + if (Files.exists(path)) { + String user = new String(Files.readAllBytes(path), StandardCharsets.UTF_8).trim(); + return user.isEmpty() ? null : user; + } + } catch (Exception e) { + logger.warn("Could not read rejected-login marker: {}", e.getMessage()); + } + return null; + } + + private void saveRejectedUser(String user) { + try { + Path dir = Paths.get(tokenDir); + Files.createDirectories(dir); + Path path = dir.resolve(REJECTED_FILE_NAME); + Files.write(path, user.getBytes(StandardCharsets.UTF_8)); + setPermissions(path, "rw-------"); + } catch (Exception e) { + logger.warn("Could not save rejected-login marker: {}", e.getMessage()); + } + } + + private void deleteRejectedUser() { + try { + Files.deleteIfExists(Paths.get(tokenDir, REJECTED_FILE_NAME)); + } catch (Exception e) { + logger.warn("Could not delete rejected-login marker: {}", e.getMessage()); + } + } + private static void setPermissions(Path path, String perms) { try { Files.setPosixFilePermissions(path, PosixFilePermissions.fromString(perms)); diff --git a/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java b/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java index e075e387..c10e6230 100644 --- a/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java +++ b/src/test/java/com/iemr/mmu/service/benid/BenIdAutoTopUpServiceTest.java @@ -78,6 +78,7 @@ void setUp() throws Exception { ReflectionTestUtils.setField(service, "lowerLimit", 2000L); ReflectionTestUtils.setField(service, "batchSize", 1000L); ReflectionTestUtils.setField(service, "bootstrapSize", 10000L); + ReflectionTestUtils.setField(service, "maxPerRequest", 1000L); ReflectionTestUtils.setField(service, "dataDir", dataDir.toString()); ReflectionTestUtils.setField(service, "retryDelaysMs", new long[] { 0L }); ReflectionTestUtils.setField(service, "connectTimeoutMs", 1000); @@ -142,8 +143,9 @@ void doesNothingWhenDisabled() throws Exception { @Test void emptyPoolGetsBootstrapSize() { + ReflectionTestUtils.setField(service, "bootstrapSize", 800L); expectCount(0); - expectGenerate(10000, 3); + expectGenerate(800, 3); expectImportOk(); expectCount(3); run(); @@ -192,8 +194,9 @@ void centralErrorLeavesNothingBehind() { @Test void failedLocalImportKeepsIdsAndImportsThemNextRun() throws Exception { + ReflectionTestUtils.setField(service, "bootstrapSize", 800L); expectCount(0); - expectGenerate(10000, 3); + expectGenerate(800, 3); server.expect(requestTo(IMPORT)).andRespond(withStatus(HttpStatus.INTERNAL_SERVER_ERROR)); // next run: pending IDs are imported first, then the pool is healthy expectImportOk(); @@ -222,4 +225,48 @@ void skipsWhileManualGenerationRuns() { BenIdAutoTopUpService.GENERATION_LOCK.unlock(); } } + + @Test + void bigDownloadIsSplitIntoRequestsOfAtMost1000() { + ReflectionTestUtils.setField(service, "bootstrapSize", 2500L); + expectCount(0); + expectGenerate(1000, 2); + expectImportOk(); + expectGenerate(1000, 2); + expectImportOk(); + expectGenerate(500, 2); + expectImportOk(); + expectCount(2500); + run(); + server.verify(); + } + + @Test + void pendingFileThatKeepsFailingIsSetAsideAndTopUpsContinue() throws Exception { + Files.writeString(pending(), "[{\"benRegId\":12"); // half-written by a power cut + for (int i = 0; i < 3; i++) { + server.expect(requestTo(IMPORT)).andRespond(withStatus(HttpStatus.INTERNAL_SERVER_ERROR)); + } + expectCount(2500); + run(); + run(); + assertTrue(Files.exists(pending()), "kept while retries are left"); + run(); + server.verify(); + assertFalse(Files.exists(pending()), "set aside after 3 failures"); + try (java.util.stream.Stream files = Files.list(dataDir)) { + assertTrue(files.anyMatch(f -> f.getFileName().toString().startsWith("benid-pending-import.failed-"))); + } + } + + @Test + void retryWaitIsSpreadPlusMinus20Percent() { + ReflectionTestUtils.setField(service, "retryDelaysMs", new long[] { 10000L }); + for (int i = 0; i < 20; i++) { + long before = System.currentTimeMillis(); + ReflectionTestUtils.invokeMethod(service, "onFailure", new RuntimeException("down")); + long wait = (long) ReflectionTestUtils.getField(service, "nextAttemptAt") - before; + assertTrue(wait >= 7990 && wait <= 12010, "wait " + wait + " ms is outside 8-12 s"); + } + } } diff --git a/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java b/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java index 744dd16e..d6ba45a0 100644 --- a/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java +++ b/src/test/java/com/iemr/mmu/service/benid/CentralAuthServiceTest.java @@ -242,4 +242,35 @@ private String assertDoesNotThrowValue() { throw new AssertionError("revoked token must not throw", e); } } + + @Test + void rejectedPasswordIsRememberedAfterRestart() { + expectLoginRejected(); + service.capture("stoptb1", "wrong", false); + central.verify(); + + CentralAuthService restarted = newService(true, 403); + // laptop login after the restart must not reach central with the rejected password + assertFalse(restarted.capture("stoptb1", "wrong", false)); + central.verify(); + } + + @Test + void successfulReconnectClearsTheRejectedMarker() { + expectLoginRejected(); + expectLoginOk("RT-9"); + service.capture("stoptb1", "wrong", false); + assertTrue(service.capture("stoptb1", "central-pw", true)); + assertFalse(Files.exists(tokenDir.resolve("central-login-rejected"))); + central.verify(); + } + + @Test + void rejectedMarkerHoldsNoPassword() throws Exception { + expectLoginRejected(); + service.capture("stoptb1", "secret-pw", false); + String marker = Files.readString(tokenDir.resolve("central-login-rejected")); + assertEquals("stoptb1", marker.trim()); + assertFalse(marker.contains("secret-pw")); + } } From b1887cb09e507ebed3f840ec6f4eae1ed55e96a7 Mon Sep 17 00:00:00 2001 From: vishwab1 Date: Wed, 14 Oct 2026 16:44:54 +0530 Subject: [PATCH 79/81] chore: set MMU-API version to 3.12.0 Co-Authored-By: Claude Opus 5.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc78ec59..5749c203 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.iemr.mmu mmu-api - 3.9.2 + 3.12.0 war MMU-API From bbe70ed60f0ce0c095477634c3086606a8ab90a7 Mon Sep 17 00:00:00 2001 From: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:15:20 +0530 Subject: [PATCH 80/81] =?UTF-8?q?StopTB=20camp=20laptop=20data=20sync=20?= =?UTF-8?q?=E2=80=93=20=3D=20sync,=20inventory=20upload=20fixes=20and=20ma?= =?UTF-8?q?ster=20download=20(#201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(AMM-2491): automatic data sync and sync failure review for StopTB laptops - Upload (van to central) every 30 min and master download every 6 h, using the central connection of the automatic BenID download (no stored password; SyncedBy = operator who connected the laptop). Own thread and lock; skips while a manual master download runs. - GET /dataSyncActivity/autoSyncStatus: last runs, last error, and rows central rejected (Processed='F') per table and SyncFailureReason. - Active only where stoptb.enforce.vanid=true and stoptb.van.id>0; manual Upload / Download Masters unchanged. 6 unit tests. Co-Authored-By: Claude Opus 5.5 * fix(AMM-2491): send sync failure time as text Co-Authored-By: Claude Opus 5.5 * feat(AMM-2491): automatic inventory data download after each upload Refills and stock issued at central for this van's store (indents, indent issues, stock transfers, stock entries) reach the laptop without pressing Inventory Data Download. Runs after each automatic upload; its own failure does not undo the upload; shown in autoSyncStatus. Switch: sync.autoInventoryDownload.enabled (default true). 2 new unit tests. Co-Authored-By: Claude Opus 5.5 * fix(datasync): inventory data download skipped indent orders and stock transfers The loop over the 5 tables also did i++ at the end of each pass, so only t_indent, t_indentissue and t_itemstockentry were downloaded; t_indentorder and t_stocktransfer never reached the van. Co-Authored-By: Claude Opus 5.5 * fix(datasync): central keeps the van row's VanSerialNo and store on upload m_synctabledetail maps van columns to server columns by position (the van's own ID sits where the server list has VanSerialNo); since the by-name record mapping, VanSerialNo was stored NULL, so a re-sent row was inserted again (duplicates) and joins on VanSerialNo broke. VanSerialNo now takes the van row ID and SyncFacilityID the row's store when not sent by name (dataSync.fillVanKeys, default true). Master download on central logs the row count instead of every row (the 661k-row village table exhausted the heap). Co-Authored-By: Claude Opus 5.5 * feat(AMM-2491): StopTB laptop master download and automatic sync off by default - StopTB laptops (stoptb.enforce.vanid=true): service-line and van masters (hubs, vans, stores, items, item-store mapping, roles) are downloaded in full for the van's line, so a laptop set up from a copied database gets them all; shared masters (villages) stay incremental. Other MMU laptops unchanged. - Automatic upload / master download / inventory download are off by default (manual sync until approved); when on, the automatic master download fetches only the inventory masters (sync.autoMasterDownload.inventoryOnly=true). Co-Authored-By: Claude Opus 5.5 * fix(datasync): central links uploaded rows to central visit/prescription IDs A van sends its own IDs in BenVisitID / PrescriptionID; on central those numbers belong to other rows, so prescriptions and prescribed drugs were linked to another van's visit/prescription, or rejected by the foreign key when the number did not exist. With dataSync.translateVanLinks=true the link is replaced by the central ID of the row the same van uploaded with that VanSerialNo; if that row is not on central yet the record is not saved and is sent again on the next upload. Off by default (other MMU servers unchanged); to be switched on for the StopTB central. Co-Authored-By: Claude Opus 5.5 * fix(datasync): StopTB laptop uploads stock tables only through the Inventory group Inventory tables are configured in group 5 (old lists: stock-outs without FacilityID/SyncFacilityID, patient issues without SyncFacilityID, batch insert fails) and in group 9 Inventory (complete). Group 5 runs first and marks the rows uploaded, so central stored stock-outs without their camp store and per-store reports missed them. On StopTB laptops (stoptb.enforce.vanid=true) a table that a later group also uploads is skipped. Other MMU laptops unchanged. Co-Authored-By: Claude Opus 5.5 * fix(datasync): StopTB laptop upload uses the renamed reason_to_close column and continues past a failing table The laptop sync list still names tb_diagnostic_order.reason_for_refusal; V125 renamed it to reason_to_close and the whole upload stopped with bad SQL grammar. The column name the table really has is used for the van and server column lists, and a table that cannot be read is recorded as failed while the other tables still upload. Both only on a StopTB laptop (stoptb.enforce.vanid). Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../environment/common_example.properties | 14 + .../dataSyncActivity/StartSyncActivity.java | 17 ++ .../SyncUtilityClassRepo.java | 2 + .../dataSyncActivity/AutoSyncService.java | 239 ++++++++++++++++++ .../dataSyncActivity/DataSyncRepository.java | 50 ++++ .../DownloadDataFromServerImpl.java | 30 +++ ...wnloadDataFromServerTransactionalImpl.java | 2 - .../UploadDataToServerImpl.java | 89 ++++++- .../DataSyncRepositoryCentral.java | 16 ++ .../DataSyncRepositoryCentralDownload.java | 3 +- .../GetDataFromVanAndSyncToDBImpl.java | 83 +++++- .../dataSyncActivity/AutoSyncServiceTest.java | 176 +++++++++++++ .../DownloadMastersFullForLineTest.java | 49 ++++ .../UploadRenamedColumnTest.java | 48 ++++ .../UploadSkipDuplicateGroupTest.java | 51 ++++ .../GetDataFromVanAndSyncToDBVanKeysTest.java | 71 ++++++ ...GetDataFromVanAndSyncToDBVanLinksTest.java | 101 ++++++++ 17 files changed, 1032 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java create mode 100644 src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java diff --git a/src/main/environment/common_example.properties b/src/main/environment/common_example.properties index 63dd2fa8..46996a54 100644 --- a/src/main/environment/common_example.properties +++ b/src/main/environment/common_example.properties @@ -135,3 +135,17 @@ benid.pool.bootstrapSize=10000 benid.token.dir=/opt/amrit/data benid.autoTopUp.initialDelayMs=120000 benid.autoTopUp.checkIntervalMs=300000 + +## Automatic Data Sync (StopTB laptops only; needs the BenID central connection above). Off by default: manual sync only. +sync.autoUpload.enabled=false +sync.autoUpload.intervalMs=1800000 +sync.autoMasterDownload.enabled=false +sync.autoMasterDownload.intervalMs=21600000 +# automatic master download fetches only inventory masters (items, stores, item->store mapping); false = all masters +sync.autoMasterDownload.inventoryOnly=true +sync.autoInventoryDownload.enabled=false + +## Central upload: replace van IDs in link columns (BenVisitID, PrescriptionID) by central IDs. +## Off by default (other MMU servers unchanged); on for the StopTB central. +## Env alternative: DATASYNC_TRANSLATEVANLINKS=true +dataSync.translateVanLinks=false diff --git a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java index 76a9477b..973fdf41 100644 --- a/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java +++ b/src/main/java/com/iemr/mmu/controller/dataSyncActivity/StartSyncActivity.java @@ -41,6 +41,7 @@ import com.iemr.mmu.service.dataSyncActivity.DownSyncDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerImpl; import com.iemr.mmu.service.dataSyncActivity.DownloadDataFromServerTransactionalImpl; +import com.iemr.mmu.service.dataSyncActivity.AutoSyncService; import com.iemr.mmu.service.dataSyncActivity.UploadDataToServerImpl; import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentFetchService; import com.iemr.mmu.service.dataSyncLayerCentral.DiagnosticDocumentPushServiceImpl; @@ -75,6 +76,9 @@ public class StartSyncActivity { @Autowired private BenIdAutoTopUpService benIdAutoTopUpService; + @Autowired + private AutoSyncService autoSyncService; + private static final String GROUP_ID = "groupID"; private static final String PROVIDER_SERVICE_MAP_ID = "providerServiceMapID"; @@ -205,6 +209,19 @@ public String startMasterDownload(@RequestBody String requestOBJ, return response.toString(); } + @Operation(summary = "Automatic data sync status and the rows central rejected on upload") + @GetMapping(value = { "/autoSyncStatus" }) + public String autoSyncStatus() { + OutputResponse response = new OutputResponse(); + try { + response.setResponse(autoSyncService.status().toString()); + } catch (Exception e) { + logger.error("Error while getting automatic sync status : " + e); + response.setError(e); + } + return response.toString(); + } + @Operation(summary = "Data sync master download progress check") @GetMapping(value = { "/checkMastersDownloadProgress" }) public String checkMastersDownloadProgress() { diff --git a/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java index c146a7a9..dd054c41 100644 --- a/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java +++ b/src/main/java/com/iemr/mmu/repo/syncActivity_syncLayer/SyncUtilityClassRepo.java @@ -31,4 +31,6 @@ @Repository public interface SyncUtilityClassRepo extends CrudRepository { List findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(Integer groupID, Boolean deleted); + + List findByDeletedOrderBySyncTableDetailID(Boolean deleted); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java new file mode 100644 index 00000000..e8971fe7 --- /dev/null +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncService.java @@ -0,0 +1,239 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.ReentrantLock; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; +import com.iemr.mmu.service.benid.CentralAuthService; + +import jakarta.annotation.PreDestroy; + +/** + * StopTB laptops: runs Data Sync by itself, so central stays current and laptops get master changes without + * anyone pressing Upload or Download Masters. + * + * - Upload (van to central) every sync.autoUpload.intervalMs (default 30 min). + * - Master download (central to van) every sync.autoMasterDownload.intervalMs (default 6 h). + * + * Uses the central connection of the automatic BenID download (CentralAuthService): no password is stored, and + * SyncedBy is the operator who connected the laptop. Active only where CentralAuthService is enabled + * (stoptb.enforce.vanid=true and stoptb.van.id>0); central and other deployments are unaffected. The manual + * buttons keep working as before. + * + * Off by default (manual sync only) until the automatic schedule is approved: set sync.autoUpload.enabled, + * sync.autoMasterDownload.enabled and sync.autoInventoryDownload.enabled to true to switch it on. + */ +@Service +public class AutoSyncService { + + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + + /** Upload and master download never run at the same time from here. */ + private static final ReentrantLock SYNC_LOCK = new ReentrantLock(); + + @Autowired + private CentralAuthService centralAuthService; + + @Autowired + private UploadDataToServerImpl uploadDataToServerImpl; + + @Autowired + private DownloadDataFromServerImpl downloadDataFromServerImpl; + + @Autowired + private DownloadDataFromServerTransactionalImpl downloadDataFromServerTransactional; + + @Autowired + private DataSyncRepository dataSyncRepository; + + @Autowired + private BenIdAutoTopUpService benIdAutoTopUpService; + + @Value("${sync.autoUpload.enabled:false}") + private boolean uploadEnabled; + + @Value("${sync.autoMasterDownload.enabled:false}") + private boolean masterDownloadEnabled; + + // refills / stock issued at central for this van's store ("Inventory Data Download"), after each upload + @Value("${sync.autoInventoryDownload.enabled:false}") + private boolean inventoryDownloadEnabled; + + // automatic master download fetches only the inventory masters (items, stores, item->store mapping, Min Stock); + // other masters (users, complaints, villages) come with the manual Download Masters button + @Value("${sync.autoMasterDownload.inventoryOnly:true}") + private boolean masterDownloadInventoryOnly; + + static final Set INVENTORY_MASTERS = Set.of("m_item", "m_itemcategory", "m_itemissueconfig", "m_uom", + "m_manufacturer", "m_pharmacologicalcategory", "m_supplier", "m_facilitytype", "m_facility", + "m_itemfacilitymapping"); + + private volatile long lastUploadAt; + private volatile String lastUploadResult; + private volatile long lastMasterDownloadAt; + private volatile String lastMasterDownloadResult; + private volatile long lastInventoryDownloadAt; + private volatile String lastInventoryDownloadResult; + private volatile String lastError; + + // own thread: a long upload must not hold the shared scheduler thread + private final ExecutorService background = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "auto-data-sync"); + t.setDaemon(true); + return t; + }); + + @PreDestroy + public void shutdown() { + background.shutdownNow(); + } + + @Scheduled(initialDelayString = "${sync.autoUpload.initialDelayMs:600000}", fixedDelayString = "${sync.autoUpload.intervalMs:1800000}") + public void scheduledUpload() { + if (uploadEnabled && centralAuthService.isEnabled()) { + background.submit(this::uploadOnce); + } + } + + @Scheduled(initialDelayString = "${sync.autoMasterDownload.initialDelayMs:900000}", fixedDelayString = "${sync.autoMasterDownload.intervalMs:21600000}") + public void scheduledMasterDownload() { + if (masterDownloadEnabled && centralAuthService.isEnabled()) { + background.submit(this::masterDownloadOnce); + } + } + + void uploadOnce() { + if (!SYNC_LOCK.tryLock()) { + return; + } + try { + String key = centralAuthService.getCentralSessionKey(); + if (key == null) { + lastError = "No central connection - waiting for a user login"; + return; + } + int vanID = centralAuthService.getVanID(); + String user = centralAuthService.getConnectedUser(); + String result = uploadDataToServerImpl.getDataToSyncToServer(vanID, + user != null && !user.isBlank() ? user : "auto-sync", key, null); + lastUploadAt = System.currentTimeMillis(); + lastUploadResult = result; + lastError = null; + logger.info("Automatic upload for vanID {} finished: {}", vanID, result); + // internet is up: top up the BenID pool if it is low (same as the manual upload) + benIdAutoTopUpService.triggerNow(); + if (inventoryDownloadEnabled) + inventoryDownload(vanID, key); + } catch (Exception e) { + lastError = "Upload: " + e.getMessage(); + logger.warn("Automatic upload failed: {}", e.getMessage()); + } finally { + SYNC_LOCK.unlock(); + } + } + + // Same as the "Inventory Data Download" button: indents, indent issues, stock transfers and stock entries + // that central has for this van's store. Its own failure does not undo the upload. + private void inventoryDownload(int vanID, String key) { + try { + int done = downloadDataFromServerTransactional.downloadTransactionalData(vanID, key, "datasync"); + lastInventoryDownloadAt = System.currentTimeMillis(); + lastInventoryDownloadResult = done > 0 ? "success" : "nothing downloaded"; + logger.info("Automatic inventory download for vanID {} finished: {}", vanID, lastInventoryDownloadResult); + } catch (Exception e) { + lastError = "Inventory download: " + e.getMessage(); + logger.warn("Automatic inventory download failed: {}", e.getMessage()); + } + } + + void masterDownloadOnce() { + if (!SYNC_LOCK.tryLock()) { + return; + } + try { + String key = centralAuthService.getCentralSessionKey(); + if (key == null) { + lastError = "No central connection - waiting for a user login"; + return; + } + int vanID = centralAuthService.getVanID(); + Integer psm = dataSyncRepository.getVanProviderServiceMapID(vanID); + if (psm == null) { + lastError = "Master download: van " + vanID + " not found in m_van"; + return; + } + String result = downloadDataFromServerImpl.downloadMasterDataFromServer(key, null, vanID, psm, + masterDownloadInventoryOnly ? INVENTORY_MASTERS : null); + if ("inProgress".equalsIgnoreCase(result)) { + logger.info("Master download already running (manual) - skipping the automatic one"); + return; + } + lastMasterDownloadAt = System.currentTimeMillis(); + lastMasterDownloadResult = result; + lastError = null; + logger.info("Automatic master download for vanID {} finished: {}", vanID, result); + } catch (Exception e) { + lastError = "Master download: " + e.getMessage(); + logger.warn("Automatic master download failed: {}", e.getMessage()); + } finally { + SYNC_LOCK.unlock(); + } + } + + /** For the Data Sync screen: last runs and the rows central has rejected. */ + public JSONObject status() { + JSONObject s = new JSONObject(); + s.put("enabled", centralAuthService.isEnabled()); + s.put("uploadEnabled", uploadEnabled); + s.put("masterDownloadEnabled", masterDownloadEnabled); + s.put("lastUploadAt", lastUploadAt); + s.put("lastUploadResult", lastUploadResult == null ? JSONObject.NULL : lastUploadResult); + s.put("lastMasterDownloadAt", lastMasterDownloadAt); + s.put("lastMasterDownloadResult", + lastMasterDownloadResult == null ? JSONObject.NULL : lastMasterDownloadResult); + s.put("inventoryDownloadEnabled", inventoryDownloadEnabled); + s.put("lastInventoryDownloadAt", lastInventoryDownloadAt); + s.put("lastInventoryDownloadResult", + lastInventoryDownloadResult == null ? JSONObject.NULL : lastInventoryDownloadResult); + s.put("lastError", lastError == null ? JSONObject.NULL : lastError); + List> failures = dataSyncRepository.getSyncFailures(); + long failedRows = failures.stream().mapToLong(f -> ((Number) f.get("failedRows")).longValue()).sum(); + s.put("failedRows", failedRows); + s.put("failures", new JSONArray(failures)); + return s; + } +} diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index 75a74842..f4850923 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -23,9 +23,11 @@ import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.sql.PreparedStatement; @@ -40,6 +42,7 @@ import org.springframework.jdbc.support.KeyHolder; import org.springframework.stereotype.Service; +import com.iemr.mmu.data.syncActivity_syncLayer.SyncUtilityClass; import com.iemr.mmu.repo.syncActivity_syncLayer.SyncUtilityClassRepo; import com.iemr.mmu.utils.validator.SqlIdentifierValidator; @@ -127,6 +130,42 @@ public int updateProcessedFlagInVan(String schemaName, String tableName, List psm = getJdbcTemplate().queryForList( + "SELECT ProviderServiceMapID FROM db_iemr.m_van WHERE VanID = ? AND Deleted = false", Integer.class, + vanID); + return psm.isEmpty() ? null : psm.get(0); + } + + /** + * Rows central rejected on upload (Processed = 'F'), per sync table and reason. They are sent again on every + * upload until central accepts them. + */ + public List> getSyncFailures() { + jdbcTemplate = getJdbcTemplate(); + List> result = new ArrayList<>(); + Set seen = new HashSet<>(); + for (SyncUtilityClass table : syncutilityClassRepo.findByDeletedOrderBySyncTableDetailID(false)) { + if (Boolean.TRUE.equals(table.getIsMaster()) || table.getSchemaName() == null + || table.getTableName() == null) + continue; + String qualified = SqlIdentifierValidator.validatedSchemaName(table.getSchemaName()) + "." + + SqlIdentifierValidator.validatedTableName(table.getTableName()); + if (!seen.add(qualified.toLowerCase())) + continue; + try { + List> rows = jdbcTemplate.queryForList("SELECT '" + table.getTableName() + + "' AS tableName, SyncFailureReason AS reason, COUNT(*) AS failedRows, DATE_FORMAT(MAX(SyncedDate), '%d-%m-%Y %H:%i') AS lastTried " + + "FROM " + qualified + " WHERE Processed = 'F' GROUP BY SyncFailureReason"); + result.addAll(rows); + } catch (Exception e) { + logger.warn("Sync failure check skipped for {}: {}", qualified, e.getMessage()); + } + } + return result; + } + // ---------------------------------- End of Upload repository // ---------------------------------- Download Repository @@ -176,6 +215,17 @@ public String resolveLastModColumn(String schema, String table) { return (found == null || found.isEmpty()) ? null : found.get(0); } + public Set getTableColumns(String schema, String table) { + jdbcTemplate = getJdbcTemplate(); + + String query = " SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? "; + + Set columns = new HashSet<>(); + for (String c : jdbcTemplate.queryForList(query, String.class, schema, table)) + columns.add(c.toLowerCase()); + return columns; + } + public List getDownSyncColumns(String schema, String table) { jdbcTemplate = getJdbcTemplate(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java index 3ad4eaa8..572724be 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerImpl.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -62,6 +63,9 @@ public class DownloadDataFromServerImpl implements DownloadDataFromServer { // rest URLs from server to download master data for van @Value("${dataSyncDownloadUrl}") private String dataSyncDownloadUrl; + // StopTB camp laptop (laptops are set up from copied databases); other MMU deployments keep standard behaviour + @Value("${stoptb.enforce.vanid:false}") + private boolean stopTBLaptop; @Autowired private SyncDownloadMasterRepo syncDownloadMasterRepo; @Autowired @@ -97,6 +101,14 @@ public class DownloadDataFromServerImpl implements DownloadDataFromServer { */ public String downloadMasterDataFromServer(String ServerAuthorization, String jwtToken, Integer vanID, Integer psmID) throws Exception { + return downloadMasterDataFromServer(ServerAuthorization, jwtToken, vanID, psmID, null); + } + + /** + * @param onlyTables master tables to download (case-insensitive), or null for all of m_downloadmastertable + */ + public String downloadMasterDataFromServer(String ServerAuthorization, String jwtToken, Integer vanID, Integer psmID, + Set onlyTables) throws Exception { if (totalCounter != progressCounter) { return "inProgress"; @@ -105,6 +117,10 @@ public String downloadMasterDataFromServer(String ServerAuthorization, String jw String successFlag = " Master download started "; ArrayList downloadMasterList = syncDownloadMasterRepo.getDownloadTables(); + if (onlyTables != null && downloadMasterList != null) { + downloadMasterList.removeIf(t -> t.getTableName() == null + || !onlyTables.contains(t.getTableName().toLowerCase())); + } totalCounter = downloadMasterList.size(); progressCounter = 0; @@ -180,6 +196,8 @@ public String call() { private int downloadDataFromServer(SyncDownloadMaster syncDownloadMaster,String ServerAuthorization, String jwtToken) throws Exception { int successFlag = 0; + if (stopTBLaptop) + fullDownloadForLineAndVanMasters(syncDownloadMaster); // initializing RestTemplate RestTemplate restTemplate = new RestTemplate(); // Provide the required second argument, e.g., an empty string or appropriate authorization token @@ -212,6 +230,18 @@ private int downloadDataFromServer(SyncDownloadMaster syncDownloadMaster,String return successFlag; } + /** + * StopTB laptops only. Service-line (P) and van (V) masters - hubs, vans, service points, stores, items, item-store mapping, roles - + * are always downloaded in full for this van's line: they are small, and a laptop set up from a copied database + * can carry download dates newer than central rows it never received (missing hub -> its vans fail on the + * foreign key). Rows are inserted/updated by ID, so repeating them is harmless. Masters shared by all lines (A, + * e.g. 661k villages) stay incremental. + */ + static void fullDownloadForLineAndVanMasters(SyncDownloadMaster table) { + if ("P".equalsIgnoreCase(table.getMasterType()) || "V".equalsIgnoreCase(table.getMasterType())) + table.setLastDownloadDate(null); + } + private int updateMastersWithLatestData(MasterDownloadDataDigester masterDownloadDataDigester, SyncDownloadMaster syncDownloadMaster) { int successFlag = 0; diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java index ee3c39a5..e441ea6e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownloadDataFromServerTransactionalImpl.java @@ -227,8 +227,6 @@ public int downloadTransactionalData(int vanID, String ServerAuthorization, Stri default: } - - i++; } return 1; diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java index d9567861..b0aa115d 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/UploadDataToServerImpl.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.json.JSONArray; import org.json.JSONException; @@ -80,6 +81,11 @@ public class UploadDataToServerImpl implements UploadDataToServer { @Value("${BATCH_SIZE}") private int BATCH_SIZE; + // StopTB camp laptop: a table configured in several sync groups is uploaded only by its last group (see + // skipTablesUploadedByLaterGroup) + @Value("${stoptb.enforce.vanid:false}") + private boolean stopTBLaptop; + @Autowired private DataSyncRepository dataSyncRepository; @Autowired @@ -151,9 +157,33 @@ private String startDataSync(int vanID, String user, String Authorization, Strin String tableKey = obj.getSchemaName() + "." + obj.getTableName(); boolean tableHasError = false; - // get data from DB to sync to server - List> syncData = getDataToSync(obj.getSchemaName(), obj.getTableName(), - obj.getVanColumnName()); + String[] columns = { obj.getVanColumnName(), obj.getServerColumnName() }; + List> syncData; + try { + if (stopTBLaptop) + columns = useRenamedColumns(obj.getTableName(), columns[0], columns[1], + dataSyncRepository.getTableColumns(obj.getSchemaName(), obj.getTableName())); + + // get data from DB to sync to server + syncData = getDataToSync(obj.getSchemaName(), obj.getTableName(), columns[0]); + } catch (Exception e) { + if (!stopTBLaptop) + throw e; + // StopTB camp laptop: one table that cannot be read must not stop the upload of the others; + // its rows stay unprocessed and go on the next upload + logger.error("Upload of table {} skipped: {}", tableKey, e.getMessage()); + groupHasFailures = true; + Map tableDetails = new HashMap<>(); + tableDetails.put("tableName", obj.getTableName()); + tableDetails.put("schemaName", obj.getSchemaName()); + tableDetails.put("status", "failed"); + tableDetails.put("totalRecords", 0); + tableDetails.put("successfulRecords", 0); + tableDetails.put("failedRecords", 0); + tableDetails.put("failureReasons", List.of("Table could not be read: " + e.getMessage())); + tableDetailsList.add(tableDetails); + continue; + } logger.info("Fetched syncData for schema {} and table {}: {}", obj.getSchemaName(), obj.getTableName(), objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(syncData)); @@ -179,7 +209,7 @@ private String startDataSync(int vanID, String user, String Authorization, Strin Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), obj.getTableName(), - obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, + obj.getVanAutoIncColumnName(), columns[1], syncDataBatch, user, Authorization, token); if (syncResult == null) { @@ -221,7 +251,7 @@ private String startDataSync(int vanID, String user, String Authorization, Strin Map syncResult = syncDataToServer(vanID, obj.getSchemaName(), obj.getTableName(), - obj.getVanAutoIncColumnName(), obj.getServerColumnName(), syncDataBatch, user, + obj.getVanAutoIncColumnName(), columns[1], syncDataBatch, user, Authorization, token); if (syncResult == null) { @@ -407,10 +437,59 @@ private String startDataSync(int vanID, String user, String Authorization, Strin private List getVanAndServerColumns(Integer groupID) throws Exception { List syncUtilityClassList = getVanAndServerColumnList(groupID); + if (stopTBLaptop) + syncUtilityClassList = skipTablesUploadedByLaterGroup(syncUtilityClassList, groupID, + syncutilityClassRepo.findByDeletedOrderBySyncTableDetailID(false)); logger.debug("Fetched SyncUtilityClass list for groupID {}: {}", groupID, syncUtilityClassList); return syncUtilityClassList; } + /** + * Inventory tables are configured twice: group 5 (old lists without the store columns of stock-outs and patient + * issues, batch insert fails) and group 9 "Inventory" (complete). Group 5 ran first and marked the rows uploaded, + * so central got stock-outs without their camp store. A table that a later group also uploads is skipped here. + */ + static List skipTablesUploadedByLaterGroup(List groupTables, int groupID, + List allTables) { + Map lastGroup = new HashMap<>(); + for (SyncUtilityClass t : allTables) { + if (t.getTableName() != null && t.getSyncTableGroupID() != null) + lastGroup.merge(t.getTableName().toLowerCase(), t.getSyncTableGroupID(), Math::max); + } + List kept = new ArrayList<>(); + for (SyncUtilityClass t : groupTables) { + Integer last = t.getTableName() == null ? null : lastGroup.get(t.getTableName().toLowerCase()); + if (last == null || last <= groupID) + kept.add(t); + } + return kept; + } + + // columns renamed by DB scripts after the sync table lists were written (tb_diagnostic_order: V114 renamed + // reason_for_refusal to reason_to_close, V117 reverted it, V125 renamed it again) + private static final Map> RENAMED_COLUMNS = Map.of("tb_diagnostic_order", + Map.of("reason_for_refusal", "reason_to_close", "reason_to_close", "reason_for_refusal")); + + /** + * The sync table list may still name a column by its old name (laptop m_synctabledetail has reason_for_refusal, + * the table has reason_to_close) and the whole upload stopped with "bad SQL grammar". The name the table really + * has is used in both the van and the server column lists (central has the same DB version). + */ + static String[] useRenamedColumns(String tableName, String vanColumns, String serverColumns, + Set tableColumns) { + Map renamed = tableName == null ? null : RENAMED_COLUMNS.get(tableName.toLowerCase()); + if (renamed == null || tableColumns == null || tableColumns.isEmpty()) + return new String[] { vanColumns, serverColumns }; + for (Map.Entry r : renamed.entrySet()) { + if (!tableColumns.contains(r.getKey()) && tableColumns.contains(r.getValue())) { + String listed = "(?i)\\b" + r.getKey() + "\\b"; + vanColumns = vanColumns == null ? null : vanColumns.replaceAll(listed, r.getValue()); + serverColumns = serverColumns == null ? null : serverColumns.replaceAll(listed, r.getValue()); + } + } + return new String[] { vanColumns, serverColumns }; + } + public List getVanAndServerColumnList(Integer groupID) throws Exception { List syncUtilityClassList = syncutilityClassRepo .findBySyncTableGroupIDAndDeletedOrderBySyncTableDetailID(groupID, false); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java index 0da5bf4b..75aa52a4 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentral.java @@ -173,6 +173,22 @@ public int checkRecordIsAlreadyPresentOrNot(String schemaName, String tableName, } } + /** + * Central ID of a row a van uploaded earlier, found by the van's own ID (VanSerialNo) and the van; null if that + * row is not on central (yet). + */ + public Long findCentralIdOfVanRow(String schemaName, String tableName, String idColumn, Object vanSerialNo, + String vanID) { + if (!isValidSchemaName(schemaName) || !isValidDatabaseIdentifierCharacter(tableName) + || !isValidDatabaseIdentifierCharacter(idColumn)) { + throw new IllegalArgumentException("Invalid identifiers provided."); + } + List ids = getJdbcTemplate().queryForList("SELECT " + idColumn + " FROM " + schemaName + "." + tableName + + " WHERE VanSerialNo = ? AND VanID = ? ORDER BY " + idColumn + " DESC LIMIT 1", Long.class, + String.valueOf(vanSerialNo), vanID); + return ids.isEmpty() ? null : ids.get(0); + } + public int[] syncDataToCentralDB(String schema, String tableName, String serverColumns, String query, List syncDataList) { jdbcTemplate = getJdbcTemplate(); diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java index c76962b4..50c49c0e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/DataSyncRepositoryCentralDownload.java @@ -205,7 +205,8 @@ else if (masterType.equalsIgnoreCase("P")) { } logger.info("Select query central: " + baseQuery); logger.info("Last Downloaded Date " + lastDownloadDate); - logger.info("Result set Details: " + resultSetList); + // Row count only: logging the rows of a large master (villages, 661k rows) exhausted the heap. + logger.info("Rows returned: " + resultSetList.size()); return resultSetList; } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java index c0e508cd..4db30fff 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBImpl.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -53,6 +54,21 @@ public class GetDataFromVanAndSyncToDBImpl implements GetDataFromVanAndSyncToDB @Value("${downSync.markDeliveredOnUpSync:true}") private boolean markDownSyncedOnUpSync; + // fill VanSerialNo / SyncFacilityID from the van row when not sent by name (see fillVanKeys) + @Value("${dataSync.fillVanKeys:true}") + private boolean fillVanKeysEnabled = true; + + // replace van IDs in link columns by central IDs (see translateVanLinks); off by default, on for StopTB central + @Value("${dataSync.translateVanLinks:false}") + private boolean translateVanLinksEnabled; + + // link column -> {table it points to, that table's ID column} + private static final Map VAN_LINKS = new LinkedHashMap<>(); + static { + VAN_LINKS.put("BenVisitID", new String[] { "t_benvisitdetail", "BenVisitID" }); + VAN_LINKS.put("PrescriptionID", new String[] { "t_prescription", "PrescriptionID" }); + } + private static final Map> TABLE_GROUPS = new HashMap<>(); static { TABLE_GROUPS.put(1, @@ -372,7 +388,21 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig // CRITICAL FIX: Keep null as null, don't convert to string cleanRecord.put(cleanKey.trim(), value); } - + if (fillVanKeysEnabled) + fillVanKeys(cleanRecord, serverColumnsList, vanAutoIncColumnName); + if (translateVanLinksEnabled) { + Object linkVanID = cleanRecord.get("VanID") != null ? cleanRecord.get("VanID") : cleanRecord.get("vanID"); + String linkError = translateVanLinks(schemaName, syncTableName, cleanRecord, serverColumnsList, + linkVanID == null ? null : String.valueOf(linkVanID)); + if (linkError != null) { + syncResults.add(new SyncResult(schemaName, syncTableName, + String.valueOf(cleanRecord.get(vanAutoIncColumnName)), syncUploadDataDigester.getSyncedBy(), + false, linkError)); + overallSuccess = false; + continue; + } + } + String vanSerialNo = String.valueOf(cleanRecord.get(vanAutoIncColumnName)); // Column may be "vanID" (Stop TB tables) or "VanID" (standard tables) — check both @@ -606,6 +636,57 @@ private boolean performGenericTableSync(SyncUploadDataDigester syncUploadDataDig return overallSuccess; } + /** + * m_synctabledetail maps van columns to server columns by position: the van's own ID (VanAutoIncColumnName, + * e.g. BenVisitID, ItemStockEntryID) sits where the server list has VanSerialNo. Records are filled by name, so + * without this VanSerialNo stays NULL: the row cannot be matched on the next upload (duplicates) and joins on + * VanSerialNo break. Same for SyncFacilityID, which some van lists do not send: it is the row's store. + */ + static void fillVanKeys(Map record, List serverColumns, String vanAutoIncColumnName) { + if (hasColumn(serverColumns, "VanSerialNo") && record.get("VanSerialNo") == null + && vanAutoIncColumnName != null && record.get(vanAutoIncColumnName) != null) { + record.put("VanSerialNo", record.get(vanAutoIncColumnName)); + } + if (hasColumn(serverColumns, "SyncFacilityID") && record.get("SyncFacilityID") == null + && record.get("FacilityID") != null) { + record.put("SyncFacilityID", record.get("FacilityID")); + } + } + + /** + * A van sends its own IDs in link columns (BenVisitID, PrescriptionID). On central those numbers belong to other + * rows (other vans, central data): the link pointed to the wrong visit/prescription, or the foreign key rejected + * the row. The link is replaced by the central ID of the row the van uploaded with that VanSerialNo. If that row + * is not on central yet, the record is not saved (returned reason) and is sent again on the next upload. + * + * @return null when done, or the reason the record cannot be saved yet + */ + String translateVanLinks(String schemaName, String tableName, Map record, + List serverColumns, String vanID) { + for (Map.Entry link : VAN_LINKS.entrySet()) { + String column = link.getKey(); + String parentTable = link.getValue()[0]; + Object vanValue = record.get(column); + if (vanValue == null || vanID == null || tableName.equalsIgnoreCase(parentTable) + || !hasColumn(serverColumns, column)) + continue; + Long centralID = dataSyncRepositoryCentral.findCentralIdOfVanRow(schemaName, parentTable, + link.getValue()[1], vanValue, vanID); + if (centralID == null) + return "Linked " + parentTable + " (van ID " + vanValue + ") not on central yet; sent again on next upload"; + record.put(column, centralID); + } + return null; + } + + private static boolean hasColumn(List columns, String name) { + for (String column : columns) { + if (column.trim().equals(name)) + return true; + } + return false; + } + private void flagUpSyncedRecordsForDownSync(String schemaName, String tableName, List syncResults, Map vanIDByResultIndex) { if (!markDownSyncedOnUpSync || vanIDByResultIndex.isEmpty()) diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java new file mode 100644 index 00000000..07fa285f --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/AutoSyncServiceTest.java @@ -0,0 +1,176 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Map; + +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import com.iemr.mmu.service.benid.BenIdAutoTopUpService; +import com.iemr.mmu.service.benid.CentralAuthService; + +class AutoSyncServiceTest { + + private AutoSyncService service; + private CentralAuthService centralAuth; + private UploadDataToServerImpl upload; + private DownloadDataFromServerImpl download; + private DataSyncRepository repo; + private BenIdAutoTopUpService benId; + private DownloadDataFromServerTransactionalImpl inventory; + + @BeforeEach + void setUp() throws Exception { + centralAuth = mock(CentralAuthService.class); + upload = mock(UploadDataToServerImpl.class); + download = mock(DownloadDataFromServerImpl.class); + repo = mock(DataSyncRepository.class); + benId = mock(BenIdAutoTopUpService.class); + inventory = mock(DownloadDataFromServerTransactionalImpl.class); + when(centralAuth.isEnabled()).thenReturn(true); + when(centralAuth.getVanID()).thenReturn(403); + when(centralAuth.getConnectedUser()).thenReturn("stoptb1"); + when(centralAuth.getCentralSessionKey()).thenReturn("SESSION-KEY"); + when(repo.getVanProviderServiceMapID(403)).thenReturn(1726); + + service = new AutoSyncService(); + ReflectionTestUtils.setField(service, "centralAuthService", centralAuth); + ReflectionTestUtils.setField(service, "uploadDataToServerImpl", upload); + ReflectionTestUtils.setField(service, "downloadDataFromServerImpl", download); + ReflectionTestUtils.setField(service, "dataSyncRepository", repo); + ReflectionTestUtils.setField(service, "benIdAutoTopUpService", benId); + ReflectionTestUtils.setField(service, "downloadDataFromServerTransactional", inventory); + ReflectionTestUtils.setField(service, "inventoryDownloadEnabled", true); + ReflectionTestUtils.setField(service, "uploadEnabled", true); + ReflectionTestUtils.setField(service, "masterDownloadEnabled", true); + ReflectionTestUtils.setField(service, "masterDownloadInventoryOnly", true); + } + + @Test + void uploadUsesCentralKeyAndConnectedOperator() throws Exception { + when(upload.getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null)).thenReturn("Data successfully synced"); + + service.uploadOnce(); + + verify(upload).getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null); + verify(benId).triggerNow(); + assertEquals("Data successfully synced", service.status().get("lastUploadResult")); + } + + @Test + void uploadIsFollowedByInventoryDownload() throws Exception { + when(upload.getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null)).thenReturn("Data successfully synced"); + when(inventory.downloadTransactionalData(403, "SESSION-KEY", "datasync")).thenReturn(1); + + service.uploadOnce(); + + verify(inventory).downloadTransactionalData(403, "SESSION-KEY", "datasync"); + assertEquals("success", service.status().get("lastInventoryDownloadResult")); + } + + @Test + void inventoryDownloadFailureKeepsUploadResult() throws Exception { + when(upload.getDataToSyncToServer(403, "stoptb1", "SESSION-KEY", null)).thenReturn("Data successfully synced"); + when(inventory.downloadTransactionalData(403, "SESSION-KEY", "datasync")).thenThrow(new RuntimeException("offline")); + + service.uploadOnce(); + + assertEquals("Data successfully synced", service.status().get("lastUploadResult")); + assertEquals("Inventory download: offline", service.status().get("lastError")); + } + + @Test + void noCentralConnectionSkipsUpload() throws Exception { + when(centralAuth.getCentralSessionKey()).thenReturn(null); + + service.uploadOnce(); + + verify(upload, never()).getDataToSyncToServer(anyInt(), anyString(), anyString(), any()); + assertEquals("No central connection - waiting for a user login", service.status().get("lastError")); + } + + @Test + void masterDownloadUsesVanServiceLineAndOnlyInventoryMasters() throws Exception { + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, AutoSyncService.INVENTORY_MASTERS)) + .thenReturn("success"); + + service.masterDownloadOnce(); + + verify(download).downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, AutoSyncService.INVENTORY_MASTERS); + assertEquals("success", service.status().get("lastMasterDownloadResult")); + } + + @Test + void masterDownloadCanFetchAllMasters() throws Exception { + ReflectionTestUtils.setField(service, "masterDownloadInventoryOnly", false); + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, null)).thenReturn("success"); + + service.masterDownloadOnce(); + + verify(download).downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, null); + } + + @Test + void manualMasterDownloadRunningIsNotCountedAsARun() throws Exception { + when(download.downloadMasterDataFromServer("SESSION-KEY", null, 403, 1726, AutoSyncService.INVENTORY_MASTERS)) + .thenReturn("inProgress"); + + service.masterDownloadOnce(); + + assertEquals(0L, service.status().getLong("lastMasterDownloadAt")); + } + + @Test + void statusSumsRejectedRows() throws Exception { + when(repo.getSyncFailures()).thenReturn(List.of( + Map.of("tableName", "t_patientissue", "reason", "Duplicate", "failedRows", 2L), + Map.of("tableName", "t_prescription", "reason", "Timeout", "failedRows", 3L))); + + JSONObject status = service.status(); + + assertEquals(5L, status.getLong("failedRows")); + assertEquals(2, status.getJSONArray("failures").length()); + } + + @Test + void disabledLaptopDoesNothing() throws Exception { + when(centralAuth.isEnabled()).thenReturn(false); + + service.scheduledUpload(); + service.scheduledMasterDownload(); + + verify(centralAuth, never()).getCentralSessionKey(); + } +} diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java new file mode 100644 index 00000000..d92182b2 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/DownloadMastersFullForLineTest.java @@ -0,0 +1,49 @@ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.sql.Timestamp; + +import org.junit.jupiter.api.Test; + +import com.iemr.mmu.data.syncActivity_syncLayer.SyncDownloadMaster; + +class DownloadMastersFullForLineTest { + + private static final Timestamp COPIED_DATE = Timestamp.valueOf("2026-10-16 13:17:31"); + + private static SyncDownloadMaster table(String masterType) { + SyncDownloadMaster t = new SyncDownloadMaster(); + t.setMasterType(masterType); + t.setLastDownloadDate(COPIED_DATE); + return t; + } + + @Test + void serviceLineMastersAreDownloadedInFull() { + SyncDownloadMaster parkingPlace = table("P"); + + DownloadDataFromServerImpl.fullDownloadForLineAndVanMasters(parkingPlace); + + assertNull(parkingPlace.getLastDownloadDate()); + } + + @Test + void vanMastersAreDownloadedInFull() { + SyncDownloadMaster vanServicePoint = table("v"); + + DownloadDataFromServerImpl.fullDownloadForLineAndVanMasters(vanServicePoint); + + assertNull(vanServicePoint.getLastDownloadDate()); + } + + @Test + void mastersSharedByAllLinesStayIncremental() { + SyncDownloadMaster villages = table("A"); + + DownloadDataFromServerImpl.fullDownloadForLineAndVanMasters(villages); + + assertEquals(COPIED_DATE, villages.getLastDownloadDate()); + } +} diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java new file mode 100644 index 00000000..adbc39de --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadRenamedColumnTest.java @@ -0,0 +1,48 @@ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +import java.util.Set; + +import org.junit.jupiter.api.Test; + +class UploadRenamedColumnTest { + + private static final String VAN = "status,error_message,reason_for_refusal,push_response_json," + + "date_format(last_mod_date,'%Y-%m-%d %H:%i:%s'),vanSerialNo"; + private static final String SERVER = "status,error_message,reason_for_refusal,push_response_json,last_mod_date,vanSerialNo"; + + @Test + void oldNameInSyncListUsesTheTableName() { + Set table = Set.of("status", "error_message", "reason_to_close", "push_response_json", "last_mod_date", + "vanserialno"); + + String[] cols = UploadDataToServerImpl.useRenamedColumns("tb_diagnostic_order", VAN, SERVER, table); + + assertArrayEquals(new String[] { VAN.replace("reason_for_refusal", "reason_to_close"), + SERVER.replace("reason_for_refusal", "reason_to_close") }, cols); + } + + @Test + void tableStillWithOldNameIsUnchanged() { + Set table = Set.of("status", "error_message", "reason_for_refusal", "push_response_json"); + + assertArrayEquals(new String[] { VAN, SERVER }, + UploadDataToServerImpl.useRenamedColumns("tb_diagnostic_order", VAN, SERVER, table)); + } + + @Test + void newNameInSyncListOnOlderTable() { + String van = "status,reason_to_close", server = "status,reason_to_close"; + + assertArrayEquals(new String[] { "status,reason_for_refusal", "status,reason_for_refusal" }, + UploadDataToServerImpl.useRenamedColumns("tb_diagnostic_order", van, server, + Set.of("status", "reason_for_refusal"))); + } + + @Test + void otherTablesAreUnchanged() { + assertArrayEquals(new String[] { VAN, SERVER }, UploadDataToServerImpl.useRenamedColumns("t_tbsuspected", VAN, + SERVER, Set.of("reason_to_close"))); + } +} diff --git a/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java new file mode 100644 index 00000000..438922a5 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncActivity/UploadSkipDuplicateGroupTest.java @@ -0,0 +1,51 @@ +package com.iemr.mmu.service.dataSyncActivity; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; + +import com.iemr.mmu.data.syncActivity_syncLayer.SyncUtilityClass; + +class UploadSkipDuplicateGroupTest { + + private static SyncUtilityClass table(int group, String name) { + SyncUtilityClass t = new SyncUtilityClass(); + t.setSyncTableGroupID(group); + t.setTableName(name); + return t; + } + + private static List names(List tables) { + return tables.stream().map(SyncUtilityClass::getTableName).collect(Collectors.toList()); + } + + // as on the laptops: inventory tables in group 5 (old lists) and group 9 (complete) + private final List all = List.of(table(4, "t_Prescription"), table(5, "t_Lab_TestResult"), + table(5, "t_ItemStockEntry"), table(5, "t_ItemStockExit"), table(5, "t_PatientIssue"), + table(9, "t_itemstockentry"), table(9, "t_itemstockexit"), table(9, "t_patientissue"), + table(9, "t_stockadjustment")); + + @Test + void inventoryTablesAreSkippedInGroup5() { + List group5 = all.stream().filter(t -> t.getSyncTableGroupID() == 5).collect(Collectors.toList()); + + assertEquals(List.of("t_Lab_TestResult"), names(UploadDataToServerImpl.skipTablesUploadedByLaterGroup(group5, 5, all))); + } + + @Test + void group9StillUploadsThem() { + List group9 = all.stream().filter(t -> t.getSyncTableGroupID() == 9).collect(Collectors.toList()); + + assertEquals(names(group9), names(UploadDataToServerImpl.skipTablesUploadedByLaterGroup(group9, 9, all))); + } + + @Test + void tablesInOneGroupAreUnchanged() { + List group4 = List.of(table(4, "t_Prescription")); + + assertEquals(List.of("t_Prescription"), names(UploadDataToServerImpl.skipTablesUploadedByLaterGroup(group4, 4, all))); + } +} diff --git a/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java new file mode 100644 index 00000000..23a2a513 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanKeysTest.java @@ -0,0 +1,71 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class GetDataFromVanAndSyncToDBVanKeysTest { + + // server list of t_itemstockentry (van list has ItemStockEntryID in VanSerialNo's place) + private static final List STOCK_ENTRY_SERVER = Arrays.asList("FacilityID", "ItemID", "BatchNo", + "QuantityInHand", "VanSerialNo", "VanID", "SyncFacilityID"); + + @Test + void vanSerialNoTakesTheVanRowIdWhenNotSentByName() { + Map record = new HashMap<>(Map.of("ItemStockEntryID", 6, "FacilityID", 428, "VanID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, STOCK_ENTRY_SERVER, "ItemStockEntryID"); + + assertEquals(6, record.get("VanSerialNo")); + } + + @Test + void syncFacilityIdTakesTheRowStoreWhenNotSent() { + Map record = new HashMap<>(Map.of("ItemStockExitID", 9, "FacilityID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, Arrays.asList("ItemStockEntryID", "FacilityID", + " VanSerialNo", "SyncFacilityID"), "ItemStockExitID"); + + assertEquals(428, record.get("SyncFacilityID")); + assertEquals(9, record.get("VanSerialNo")); + } + + @Test + void valuesSentByTheVanAreKept() { + Map record = new HashMap<>( + Map.of("PatientIssueID", 3, "VanSerialNo", 3, "FacilityID", 428, "SyncFacilityID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, Arrays.asList("VanSerialNo", "SyncFacilityID"), + "PatientIssueID"); + + assertEquals(3, record.get("VanSerialNo")); + assertEquals(428, record.get("SyncFacilityID")); + } + + @Test + void columnsNotOnTheServerAreNotAdded() { + Map record = new HashMap<>(Map.of("BenVisitID", 1991, "VanID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, Arrays.asList("BeneficiaryRegID", "VanSerialNo", "VanID"), + "BenVisitID"); + + assertEquals(1991, record.get("VanSerialNo")); + assertFalse(record.containsKey("SyncFacilityID")); + } + + @Test + void missingVanIdLeavesVanSerialNoEmpty() { + Map record = new HashMap<>(Map.of("FacilityID", 428)); + + GetDataFromVanAndSyncToDBImpl.fillVanKeys(record, STOCK_ENTRY_SERVER, "ItemStockEntryID"); + + assertNull(record.get("VanSerialNo")); + } +} diff --git a/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java new file mode 100644 index 00000000..0b5e6017 --- /dev/null +++ b/src/test/java/com/iemr/mmu/service/dataSyncLayerCentral/GetDataFromVanAndSyncToDBVanLinksTest.java @@ -0,0 +1,101 @@ +package com.iemr.mmu.service.dataSyncLayerCentral; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +class GetDataFromVanAndSyncToDBVanLinksTest { + + private GetDataFromVanAndSyncToDBImpl service; + private DataSyncRepositoryCentral repo; + + @BeforeEach + void setUp() { + service = new GetDataFromVanAndSyncToDBImpl(); + repo = mock(DataSyncRepositoryCentral.class); + ReflectionTestUtils.setField(service, "dataSyncRepositoryCentral", repo); + } + + private static Map row(Object... kv) { + Map m = new HashMap<>(); + for (int i = 0; i < kv.length; i += 2) + m.put((String) kv[i], kv[i + 1]); + return m; + } + + @Test + void prescriptionGetsTheCentralIdOfItsVisit() { + // laptop visit 1261 of van 428 is central visit 5003 (central's own 1261 is another van's visit) + when(repo.findCentralIdOfVanRow("db_iemr", "t_benvisitdetail", "BenVisitID", 1261, "428")).thenReturn(5003L); + Map rx = row("PrescriptionID", 378, "BenVisitID", 1261, "VanID", 428); + + String error = service.translateVanLinks("db_iemr", "t_prescription", rx, + List.of("BenVisitID", "VanSerialNo", "VanID"), "428"); + + assertNull(error); + assertEquals(5003L, rx.get("BenVisitID")); + assertEquals(378, rx.get("PrescriptionID")); // own ID of t_prescription is not a link + } + + @Test + void prescribedDrugGetsCentralVisitAndPrescription() { + when(repo.findCentralIdOfVanRow("db_iemr", "t_benvisitdetail", "BenVisitID", 1261, "428")).thenReturn(5003L); + when(repo.findCentralIdOfVanRow("db_iemr", "t_prescription", "PrescriptionID", 378, "428")).thenReturn(9001L); + Map drug = row("PrescribedDrugID", 5, "BenVisitID", 1261, "PrescriptionID", 378); + + assertNull(service.translateVanLinks("db_iemr", "t_prescribeddrug", drug, + Arrays.asList("BenVisitID", "PrescriptionID", "VanSerialNo"), "428")); + + assertEquals(5003L, drug.get("BenVisitID")); + assertEquals(9001L, drug.get("PrescriptionID")); + } + + @Test + void parentNotOnCentralYetIsNotSaved() { + when(repo.findCentralIdOfVanRow(anyString(), anyString(), anyString(), any(), anyString())).thenReturn(null); + Map rx = row("BenVisitID", 1991); + + String error = service.translateVanLinks("db_iemr", "t_prescription", rx, List.of("BenVisitID"), "428"); + + assertNotNull(error); + assertEquals(1991, rx.get("BenVisitID")); // left as it was, row not saved + } + + @Test + void visitItselfAndRowsWithoutLinksAreUnchanged() { + Map visit = row("BenVisitID", 1261, "VisitCode", 10042800001261L); + assertNull(service.translateVanLinks("db_iemr", "t_benvisitdetail", visit, List.of("BenVisitID"), "428")); + assertEquals(1261, visit.get("BenVisitID")); + + Map batch = row("ItemStockEntryID", 41, "FacilityID", 428); + assertNull(service.translateVanLinks("db_iemr", "t_itemstockentry", batch, List.of("FacilityID"), "428")); + + Map noVan = row("BenVisitID", 1261); + assertNull(service.translateVanLinks("db_iemr", "t_prescription", noVan, List.of("BenVisitID"), null)); + verify(repo, never()).findCentralIdOfVanRow(anyString(), anyString(), anyString(), any(), anyString()); + } + + @Test + void linkColumnNotUploadedIsLeftAlone() { + Map issue = row("PrescriptionID", 378); + + assertNull(service.translateVanLinks("db_iemr", "t_patientissue", issue, List.of("VisitCode"), "428")); + + verify(repo, never()).findCentralIdOfVanRow(anyString(), anyString(), anyString(), any(), anyString()); + } +} From 6b28dbce3b23c0ae41c1118912ce3ebbb19de489 Mon Sep 17 00:00:00 2001 From: Vishwanath Balkur <118195001+vishwab1@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:29:19 +0530 Subject: [PATCH 81/81] fix(downsync): create worklist rows for down-synced beneficiaries (#203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(downsync): create worklist rows for down-synced beneficiaries Down-sync copies the beneficiary tables but not i_ben_flow_outreach, and FLW-API getBeneficiaryData serves tabs only beneficiaries that have a worklist row. Beneficiaries registered elsewhere and down-synced to the laptop therefore never reached the laptop's other tabs, so tab counts differed. After each down-sync, insert the missing worklist row for this van's beneficiaries, filled as TM-API does at registration. Processed = 'P' so it is not uploaded back (central already has its own row); modified_by = 'downsync' marks these rows. A failure here is logged and does not undo the down-sync. Co-Authored-By: Claude Opus 5.5 * fix(s3): let vans start without AWS keys — only central needs them (#202) Vans (camp laptops) run the same war as central but never call S3: they push diagnostic documents to central over HTTP. Yet startup failed with "Could not resolve placeholder 'AWS_S3_ACCESS_KEY'", and blank values would still crash in AwsBasicCredentials.create ("Access key ID cannot be blank"). - common_docker.properties: default the S3 placeholders (keys/bucket empty, region ap-south-1, push batch size 3). - S3ClientConfig: with blank keys use DefaultCredentialsProvider, which resolves lazily on first S3 call instead of at startup. Central keeps setting real AWS_S3_* values and behaves as before. Co-authored-by: Claude Opus 5.5 (cherry picked from commit 10b8ffaea199506b88811d3228877b37aaada550) --------- Co-authored-by: Claude Opus 5.5 --- src/main/environment/common_docker.properties | 10 ++-- .../com/iemr/mmu/config/S3ClientConfig.java | 13 ++++- .../dataSyncActivity/DataSyncRepository.java | 58 +++++++++++++++++++ .../DownSyncDataFromServerImpl.java | 18 ++++++ 4 files changed, 93 insertions(+), 6 deletions(-) diff --git a/src/main/environment/common_docker.properties b/src/main/environment/common_docker.properties index 38be2dce..05a361a2 100644 --- a/src/main/environment/common_docker.properties +++ b/src/main/environment/common_docker.properties @@ -6,10 +6,10 @@ spring.datasource.password=${DATABASE_PASSWORD} spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver ## S3 storage for diagnostic documents pushed from a van -aws.s3.access-key=${AWS_S3_ACCESS_KEY} -aws.s3.secret-key=${AWS_S3_SECRET_KEY} -aws.s3.region=${AWS_S3_REGION} -diagnostic.documents.s3.bucket=${DIAGNOSTIC_DOCUMENTS_S3_BUCKET} +aws.s3.access-key=${AWS_S3_ACCESS_KEY:} +aws.s3.secret-key=${AWS_S3_SECRET_KEY:} +aws.s3.region=${AWS_S3_REGION:ap-south-1} +diagnostic.documents.s3.bucket=${DIAGNOSTIC_DOCUMENTS_S3_BUCKET:} ## Carestream URLs, local carestreamOrderCreateURL=${COMMON_API}carestream/createOrder @@ -36,7 +36,7 @@ dataSyncDownloadUrl=${MMU_CENTRAL_SERVER}dataSync/server-to-van ## Diagnostic document push (this server -> further central server) diagnosticDocumentUploadUrl=${MMU_CENTRAL_SERVER}dataSync/diagnostic-documents -diagnosticDocument.push.batchSize=${DIAGNOSTIC_DOCUMENT_PUSH_BATCH_SIZE} +diagnosticDocument.push.batchSize=${DIAGNOSTIC_DOCUMENT_PUSH_BATCH_SIZE:3} diagnostic.documents.storage-root=${DIAGNOSTIC_DOCUMENTS_STORAGE_ROOT} ## TC specialist slot booking, # not required diff --git a/src/main/java/com/iemr/mmu/config/S3ClientConfig.java b/src/main/java/com/iemr/mmu/config/S3ClientConfig.java index af02dcbb..eb3509bf 100644 --- a/src/main/java/com/iemr/mmu/config/S3ClientConfig.java +++ b/src/main/java/com/iemr/mmu/config/S3ClientConfig.java @@ -5,6 +5,8 @@ import org.springframework.context.annotation.Configuration; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -26,7 +28,16 @@ public class S3ClientConfig { @Value("${aws.s3.region}") private String region; - private StaticCredentialsProvider credentialsProvider() { + /* + * Only the central server talks to S3; a van (camp laptop) runs the same war but never + * calls it, so it must be able to start without AWS keys. AwsBasicCredentials rejects blank + * keys at construction time, so fall back to the SDK default chain, which is resolved lazily + * on the first S3 call rather than at startup. + */ + private AwsCredentialsProvider credentialsProvider() { + if (accessKey == null || accessKey.isBlank() || secretKey == null || secretKey.isBlank()) { + return DefaultCredentialsProvider.create(); + } return StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)); } diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java index f4850923..99b4ec04 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DataSyncRepository.java @@ -169,6 +169,64 @@ public List> getSyncFailures() { // ---------------------------------- End of Upload repository // ---------------------------------- Download Repository + + /** + * Down-sync copies the beneficiary tables but not i_ben_flow_outreach, and the tab download + * (FLW-API getBeneficiaryData) only serves beneficiaries that have a worklist row. So a + * beneficiary registered elsewhere and down-synced here never reached this laptop's tabs. + * Creates the missing worklist row for this van's beneficiaries, filled as TM-API does at + * registration. Processed = 'P' so it is not uploaded back: central already has its own row. + */ + public int createMissingWorklistRows(int vanID, int providerServiceMapID) { + jdbcTemplate = getJdbcTemplate(); + + String village = "(SELECT h.%s FROM db_identity.i_beneficiarydetails_rmnch r " + + " JOIN db_identity.i_householddetails h ON h.houseoldId = r.houseoldId " + + " WHERE r.BeneficiaryRegID = bm.BenRegId LIMIT 1)"; + + String insert = "INSERT INTO db_iemr.i_ben_flow_outreach " + + " (beneficiary_reg_id, beneficiary_id, visit_no, ben_name, ben_age, ben_dob, ben_gender, " + + " ben_gender_val, ben_age_val, ben_phone_no, father_name, spouse_name, nurse_flag, doctor_flag, " + + " pharmacist_flag, doctor_signature_flag, BenArrivedFlag, created_by, created_date, modified_by, " + + " modified_date, registrationDate, districtID, district, villageID, village, " + + " providerServiceMapID, vanID, Processed, deleted) " + + " SELECT * FROM ( SELECT bm.BenRegId, rg.BeneficiaryID, 1 AS visit_no, " + + " TRIM(CONCAT_WS(' ', d.FirstName, NULLIF(d.MiddleName, ''), d.LastName)) AS ben_name, " + + " CONCAT(TIMESTAMPDIFF(YEAR, d.DOB, bm.CreatedDate), ' years - ', " + + " MOD(TIMESTAMPDIFF(MONTH, d.DOB, bm.CreatedDate), 12), ' months') AS ben_age, " + + " d.DOB, d.Gender, d.GenderId, TIMESTAMPDIFF(YEAR, d.DOB, bm.CreatedDate) AS ben_age_val, " + + " COALESCE(c.PreferredPhoneNum, c.PhoneNum1) AS phone, " + + " COALESCE(d.FatherName, '') AS father, COALESCE(d.SpouseName, '') AS spouse, " + + " 1 AS nf, 0 AS df, 0 AS pf, 0 AS dsf, 0 AS baf, bm.CreatedBy, bm.CreatedDate AS cdate, " + + " 'downsync' AS mby, NOW() AS mdate, bm.CreatedDate AS rdate, " + + " COALESCE(a.CurrDistrictId, a.PermDistrictId) AS districtID, " + + " COALESCE(a.CurrDistrict, a.PermDistrict) AS district, " + + " COALESCE(" + String.format(village, "villageid") + ", a.CurrVillageId, a.PermVillageId) AS villageID, " + + " COALESCE(" + String.format(village, "village") + ", a.CurrVillage, a.PermVillage) AS village, " + + " ? AS psm, bm.VanID, 'P' AS processed, 0 AS deleted " + + " FROM db_identity.i_beneficiarymapping bm " + + " JOIN db_identity.m_beneficiaryregidmapping rg ON rg.BenRegId = bm.BenRegId " + + " JOIN db_identity.i_beneficiarydetails d ON d.BeneficiaryDetailsId = bm.BenDetailsId " + + " LEFT JOIN db_identity.i_beneficiarycontacts c ON c.BenContactsID = bm.BenContactsId " + + " LEFT JOIN db_identity.i_beneficiaryaddress a ON a.BenAddressID = bm.BenAddressId " + + " WHERE bm.VanID = ? AND bm.Deleted = false " + // the van can hold beneficiaries of other service lines; only its own get a row + + " AND (SELECT r.ProviderServiceMapID FROM db_identity.i_beneficiarydetails_rmnch r " + + " WHERE r.BeneficiaryRegID = bm.BenRegId LIMIT 1) = ? " + + " AND NOT EXISTS (SELECT 1 FROM db_iemr.i_ben_flow_outreach f " + + " WHERE f.beneficiary_reg_id = bm.BenRegId) ) x " + + " WHERE x.villageID IS NOT NULL"; + + int inserted = jdbcTemplate.update(insert, providerServiceMapID, vanID, providerServiceMapID); + + if (inserted > 0) + // same convention as TM-API: VanSerialNo is the row's own id + jdbcTemplate.update("UPDATE db_iemr.i_ben_flow_outreach SET VanSerialNo = ben_flow_id " + + " WHERE modified_by = 'downsync' AND VanSerialNo IS NULL"); + + return inserted; + } + public int[] updateLatestMasterInLocal(String query, List syncDataList) { int[] i = null; // get JDBC template diff --git a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java index c3c1e1d0..2c70854e 100644 --- a/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java +++ b/src/main/java/com/iemr/mmu/service/dataSyncActivity/DownSyncDataFromServerImpl.java @@ -163,6 +163,7 @@ public String startDownSync(String serverAuthorization, String jwtToken, Integer } progressCounter++; } + createMissingWorklistRows(vanID, providerServiceMapID); countOutstandingConflicts(downSyncTables, vanID); } finally { IN_PROGRESS.set(false); @@ -207,6 +208,23 @@ public Map getDownSyncStatus() { return resultMap; } + /** Down-synced beneficiaries need a worklist row before this laptop's tabs can download them. */ + private void createMissingWorklistRows(Integer vanID, Integer providerServiceMapID) { + try { + Integer psm = providerServiceMapID != null ? providerServiceMapID + : dataSyncRepository.getVanProviderServiceMapID(vanID); + if (psm == null) { + logger.warn("Worklist rows not created after down-sync: no service line for van {}", vanID); + return; + } + int created = dataSyncRepository.createMissingWorklistRows(vanID, psm); + logger.info("Created {} worklist rows for down-synced beneficiaries of van {}", created, vanID); + } catch (Exception e) { + // the down-sync itself already succeeded; a failure here must not undo it + logger.error("Creating worklist rows after down-sync failed for van " + vanID + " : " + e.getMessage(), e); + } + } + private void countOutstandingConflicts(List downSyncTables, Integer vanID) { outstandingConflictCounter = 0; outstandingConflictTables.clear();