GitLab

at 2023-01-24T07:20:44+01:00

Pom - downgrade hsqldb to 2.7.0 (2.7.1 add some errors on uncanaccess...)

AVDTH - Review and clarify dcp migration

AVDTH - Review TRANSMITTING_BUOY_OPERATION_CODE_MAPPING mapping

AVDTH - Review OBSERVED_SYSTEM_CODE_MAPPING (for ACTIVITE.C_TYP_OBJET == 12) :
  * add Observed system 20
  * add ObjectMaterial ANLOG

AVDTH - Improve some error messages

AVDTH - Add *addMessage* method to reader

AVDTH - Improve when to create (or not) Floating object

AVDTH - Do not migrate ACTIVITE.V_POIDS_ESTIM_DCP.

AVDTH - Improve a comment

AVDTH - Add more tests (using avdth multi-year databases)

AVDTH - Improve when to create (or not) Floating object

Avdth tests - fix when less than 3 db found

Avdth tests - add missing test_forcePrepare to pom properties for avdth all years tests

Avdth - add comment (for debug purpose)

Avdth - found more ACTIVITE.C_OPERA with DCP (for 9.0.x we still import them on bad ObServe activities, will be review in 9.1.x with new vesselActivities...)

Avdth - add ACTIVITE.C_OPERA when creating ObjectMaterial to add it on log if something is wrong

         this.notReadResult = notReadResult;

         this.exportResult = exportResult;

         this.sqlResultPath = sqlResultPath;

+        // Used for debugging...

+//        if (!exportResult.isEmpty()) {

+//            System.out.println(exportResult.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("\n")));

+//        }

     }

     public Path getMessageFile() {

     protected final AvdthDataImportConfiguration getContext() {

         return context.getConfiguration();

     }

+

+    protected final void addMessage(String message) {

+        context.addMessage(message);

+    }

 }

 import fr.ird.observe.entities.referential.ps.logbook.WellContentStatus;

 import fr.ird.observe.entities.referential.ps.logbook.WellSamplingConformity;

 import fr.ird.observe.entities.referential.ps.logbook.WellSamplingStatus;

-import fr.ird.observe.persistence.avdth.data.logbook.FloatingObjectReader;

 import fr.ird.observe.persistence.avdth.data.logbook.WellPlanReader;

 import fr.ird.observe.persistence.avdth.referential.AvdthReferentialImportResult;

 import fr.ird.observe.persistence.avdth.referential.SpeciesCache;

  * @author Tony Chemit - dev@tchemit.fr

  * @since 9.0.0

  */

-@SuppressWarnings("SpellCheckingInspection")

 public class ImportReferentialContext {

-

     private Map<String, WeightCategory> landingCategories;

     private Map<String, Destination> destinations;

     private Map<String, WeightCategory> wellCategories;

     private ObjectMaterial objectMaterialDFAD;

     private ObjectMaterial objectMaterialALOG;

     private ObjectMaterial objectMaterialAFAD;

+    private ObjectMaterial objectMaterialBiodegradable;

     private ObjectMaterial objectMaterialAlone;

     private ObjectMaterial objectMaterialLOG;

     private ObjectMaterial objectMaterialFALOG;

+    private ObjectMaterial objectMaterialANLOG;

     private ObjectMaterial objectMaterialHALOG;

     private ObjectMaterial objectMaterialVNLOG;

     private ObservedSystem observedSystem0;

         objectMaterialLOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.50");

         objectMaterialFALOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.63");

         objectMaterialHALOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.67");

+        objectMaterialANLOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.56");

         objectMaterialVNLOG = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.52");

         objectMaterialAlone = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#1561561977652#0.5876332198776647");

+        objectMaterialBiodegradable = objectMaterial.get("fr.ird.referential.ps.common.ObjectMaterial#0#0.69");

         // catch

         catchCategories = new TreeMap<>();

         return objectMaterialAFAD;

     }

+    public ObjectMaterial getObjectMaterialLOG() {

+        return objectMaterialLOG;

+    }

+

+    public ObjectMaterial getObjectMaterialFALOG() {

+        return objectMaterialFALOG;

+    }

+

+    public ObjectMaterial getObjectMaterialHALOG() {

+        return objectMaterialHALOG;

+    }

+

+    public ObjectMaterial getObjectMaterialVNLOG() {

+        return objectMaterialVNLOG;

+    }

+

+    public ObjectMaterial getObjectMaterialBiodegradable() {

+        return objectMaterialBiodegradable;

+    }

+

     public ObjectMaterial getObjectMaterialAlone() {

         return objectMaterialAlone;

     }

+    public ObjectMaterial getObjectMaterialANLOG() {

+        return objectMaterialANLOG;

+    }

+

     public SampleType getLocalmarketSampleType() {

         return localmarketSampleType;

     }

         entity.addObservedSystem(getObservedSystem("102"));

     }

-    public ObjectOperation getObjectOperationByVesselActivityCode(String vesselActivityCode) {

-        String objectOperationCode = FloatingObjectReader.DCP_OBJECT_OPERATION_CODE_MAPPING.get(vesselActivityCode);

-        if (objectOperationCode == null) {

-            return null;

-        }

+    public ObjectOperation getObjectOperation(String objectOperationCode) {

         return objectOperation.get(objectOperationCode);

     }

-    public TransmittingBuoyOperation getTransmittingBuoyOperationByVesselActivityCode(String vesselActivityCode) {

-        String transmittingBuoyTypeCode = FloatingObjectReader.TRANSMITTING_BUOY_OPERATION_CODE_MAPPING.get(vesselActivityCode);

+    public TransmittingBuoyOperation getTransmittingBuoyOperation(String transmittingBuoyTypeCode) {

         if (transmittingBuoyTypeCode == null) {

             return null;

         }

         return transmittingBuoyOperation.get(transmittingBuoyTypeCode);

     }

-    public TransmittingBuoyType getTransmittingBuoyTypeByBuoyTypeCode(String buoyTypeCode) {

-        String transmittingBuoyTypeCode = FloatingObjectReader.TRANSMITTING_BUOY_TYPE_CODE_MAPPING.get(buoyTypeCode);

-        if (transmittingBuoyTypeCode == null) {

-            transmittingBuoyTypeCode = "999";

-        }

-        TransmittingBuoyType transmittingBuoyType = this.transmittingBuoyType.get(transmittingBuoyTypeCode);

-        return Objects.requireNonNull(transmittingBuoyType, String.format("Can't find transmittingBuoyType with code: %s", buoyTypeCode));

+    public TransmittingBuoyType getTransmittingBuoyType(String transmittingBuoyTypeCode) {

+        TransmittingBuoyType result = transmittingBuoyType.get(transmittingBuoyTypeCode);

+        return Objects.requireNonNull(result, String.format("Can't find TransmittingBuoyType with code: %s", transmittingBuoyTypeCode));

     }

-    public TransmittingBuoyOwnership getTransmittingBuoyOwnership(String buoyOwnershipCode) {

-        if (buoyOwnershipCode == null) {

-            return null;

-        }

-        String transmittingBuoyOwnershipCode = FloatingObjectReader.TRANSMITTING_BUOY_OWNERSHIP_CODE_MAPPING.get(buoyOwnershipCode);

+    public TransmittingBuoyOwnership getTransmittingBuoyOwnership(String transmittingBuoyOwnershipCode) {

         if (transmittingBuoyOwnershipCode == null) {

             return null;

         }

         return transmittingBuoyOwnership.get(transmittingBuoyOwnershipCode);

     }

-    public ObservedSystem getObservedSystem(String objectTypeCode, String buoyTypeCode) {

-        if (objectTypeCode.equals("999")) {

-            if (buoyTypeCode.equals("999")) {

-                return getObservedSystem0();

-            }

-            return getObservedSystem20();

-        }

-        String observedSystemCode = FloatingObjectReader.OBSERVED_SYSTEM_CODE_MAPPING.get(objectTypeCode);

-        return getObservedSystem(observedSystemCode, false);

-    }

-

-    /**

-     * To get objectMaterial id associated by a DCP from avdth ACTIVITE.C_TYP_OBJET code.

-     *

-     * @param objectTypeCode ACTIVITE.C_TYP_OBJET

-     * @return material or null

-     */

-    public ObjectMaterial getObjectMaterialByObjectTypeCode(String objectTypeCode) {

-        switch (objectTypeCode) {

-            case "1":

-                return objectMaterialAFAD;

-            case "2":

-                return objectMaterialDFAD;

-            case "3":

-                return objectMaterialLOG;

-            case "9":

-                return objectMaterialFOB;

-            case "10":

-                return objectMaterialFALOG;

-            case "11":

-                return objectMaterialHALOG;

-            case "13":

-                return objectMaterialVNLOG;

-        }

-        return null;

-    }

-

     public WeightCategory getWellWeightCategory(String weightCategoryCode) {

         String categoryCode = "W-" + weightCategoryCode;

         WeightCategory weightCategory = wellCategories.get(categoryCode);

-        return Objects.requireNonNull(weightCategory, "Can't find weight category for code: " + categoryCode);

+        return Objects.requireNonNull(weightCategory, String.format("Can't find weight category for code: %s", categoryCode));

     }

     public VesselActivity getVesselActivity6() {

         return vesselActivity6;

     }

+

 }
  */

 import com.google.common.collect.ImmutableMap;

+import fr.ird.observe.entities.data.ps.logbook.Activity;

 import fr.ird.observe.entities.data.ps.logbook.FloatingObject;

 import fr.ird.observe.entities.data.ps.logbook.FloatingObjectPart;

 import fr.ird.observe.entities.data.ps.logbook.TransmittingBuoy;

 import fr.ird.observe.entities.referential.ps.common.TransmittingBuoyOwnership;

 import fr.ird.observe.entities.referential.ps.common.TransmittingBuoyType;

 import fr.ird.observe.entities.referential.ps.common.VesselActivity;

+import fr.ird.observe.persistence.avdth.Query;

 import fr.ird.observe.persistence.avdth.data.DataReader;

 import fr.ird.observe.persistence.avdth.data.ImportDataContext;

 import fr.ird.observe.persistence.avdth.data.ImportEngine;

 import org.apache.commons.lang3.mutable.MutableInt;

+import org.apache.logging.log4j.LogManager;

+import org.apache.logging.log4j.Logger;

 import java.sql.ResultSet;

 import java.sql.SQLException;

+import java.util.List;

 import java.util.Map;

 import java.util.Objects;

 import java.util.Set;

+import java.util.TreeMap;

 import java.util.TreeSet;

 /**

  */

 @SuppressWarnings("SpellCheckingInspection")

 public class FloatingObjectReader extends DataReader<FloatingObject> {

+    private static final Logger log = LogManager.getLogger(FloatingObjectReader.class);

     /**

      * To get FloatingObject.objectOperation code from avdth ACTIVITE.C_OPERA.

      */

             .put("14", "99")

             .put("7", "99")

             .put("8", "99")

+            .put("9", "99")

+            .put("12", "99")

             .put("13", "99")

+            .put("15", "99")

             .put("5", "1")

             .put("6", "4")

             .build();

     /**

      * To get TransmittingBuoy.TransmittingBuoyOperation code from avdth ACTIVITE.C_OPERA code.

+     * <p>

+     * If for a given {@code ACTIVITE.C_OPERA}, no value is found here, this means that there is no buoy to create.

      */

     public static final Map<String, String> TRANSMITTING_BUOY_OPERATION_CODE_MAPPING = ImmutableMap.<String, String>builder()

-            .put("0", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("1", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("2", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("3", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("4", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("10", "1")// Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("14", "1")// Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("7", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("8", "1") // Visite          | 99 inconnu               | W_ARRIVING    |

-            .put("13", "1")// Visite          | 99 inconnu               | W_ARRIVING    |

-

-            .put("5", "3")  // Mise à l'eau    | null ou 3 Mise à l'eau | W_LEAVING     | Selon présence infos balise dans AVDTH

-            .put("6", "2")  // Retrait         | null ou 2 Récupération | W_ARRIVING    | Selon présence infos balise dans AVDTH

-            .put("23", "3") // Mise à l'eau    | 3 Mise à l'eau         | W_LEAVING     |

-            .put("24", "2") // Retrait         | 2 Récupération         | W_ARRIVING    |

-            .put("25", "3") // Visite          | 3 Mise à l'eau         | W_ARRIVING    |

-            .put("26", "2") // Visite          | 2 Récupération         | W_ARRIVING    |

-            .put("29", "1") // Visite          | 1 Visite               | W_ARRIVING    |

-            .put("32", "3") // Modif ou renfor | 3 Mise à l'eau         | W_ARR + W_LEA |

-            .put("33", "4") // Perte           | 4 Perte signal         | W_ARRIVING    | codes 11 et 4 à créer ; matériaux sur W_ARRIVING par convention, comme pour Visite

-            .put("34", "2") // Retrait         | 2 Récupération         | W_ARRIVING    | rattacher cette balise à un FOB de type '2-2-4-4 Bouée émettrice seule'

+            .put("5", "3")  // Mise à l'eau    | null ou 3 Mise à l'eau

+            .put("6", "2")  // Retrait         | null ou 2 Récupération

+            .put("23", "3") // Mise à l'eau    | 3 Mise à l'eau

+            .put("24", "2") // Retrait         | 2 Récupération

+            .put("25", "3") // Visite          | 3 Mise à l'eau

+            .put("26", "2") // Visite          | 2 Récupération

+            .put("29", "1") // Visite          | 1 Visite

+            .put("32", "3") // Modif ou renfor | 3 Mise à l'eau

+            .put("33", "4") // Perte           | 4 Perte signal

+            .put("34", "2") // Retrait         | 2 Récupération

             // See https://gitlab.com/ultreiaio/ird-observe/-/issues/2568

-            .put("40", "4") // Perte           | 4 Perte signal         | W_ARRIVING    | rattacher cette balise à un FOB de type '1-1 DFAD'

-            .put("41", "5") // Arret commandé de la transmission  | 5 Fin d'utilisation | W_ARRIVING    | rattacher cette balise à un FOB de type '1-1 DFAD'

+            .put("40", "4") // Perte           | 4 Perte signal

+            .put("41", "5") // Arret commandé de la transmission  | 5 Fin d'utilisation

             .build();

     /**

      * Is floatingObjectPart on leaving from avdth ACTIVITE.C_OPERA.

             "2",

             "3",

             "4",

+            "9",

             "10",

+            "12",

             "14",

+            "15",

             "7",

             "8",

             "13",

             .put("9", "20")

             .put("10", "20")

             .put("11", "20")

+            .put("12", "20")

             .put("13", "20")

             .build();

     public static final Set<String> OBSERVED_SYTEM_CODES_WITH_DCP = Set.of("20", "21", "22", "23", "24", "25");

     private final MutableInt floatingObjectPartCount = new MutableInt();

     private final MutableInt transmittingBuoyCount = new MutableInt();

+    private final Map<String, MutableInt> badVesselActivityCodes = new TreeMap<>();

+

     public FloatingObjectReader(ImportEngine context) {

         super(context);

     }

     @Override

     public FloatingObject read(ImportDataContext dataContext, ResultSet resultSet) throws SQLException {

-        String vesselActivityCode = resultSet.getString(13);

-        VesselActivity vesselActivity = dataContext.getVesselActivity(vesselActivityCode);

+        // Get the current activity where to add the optional floating object

+        Activity activity = dataContext.getActivity();

+

+        // Get his vessel activity

+        VesselActivity vesselActivity = activity.getVesselActivity();

+

+        // Avdth observed system codes associated to the current activity (need them to compute ex nihilo floating object)

         Set<String> observedSystemCodes = dataContext.getObservedSystemCodes();

-        if (!Objects.equals(vesselActivity.getCode(), "13")) {

-            boolean noDcpInObservedSystems = observedSystemCodes.stream().noneMatch(OBSERVED_SYTEM_CODES_WITH_DCP::contains);

-            if (noDcpInObservedSystems) {

-                reset(dataContext);

-                return null;

-            }

-        }

-        FloatingObject entity = newEntity(FloatingObject.SPI);

+        // We need to get the original vessel activity code from AVDTH (to compute some mapping)

+        String vesselActivityCode = resultSet.getString(13);

-        ObjectOperation objectOperation = dataContext.getObjectOperationByVesselActivityCode(vesselActivityCode);

-        entity.setObjectOperation(Objects.requireNonNull(objectOperation, String.format("Can't find objectOperation with code: %s", vesselActivityCode)));

-        boolean addCoordinate = "11".equals(objectOperation.getCode());

-        boolean whenArriving = WHEN_ARRIVING_CODE.contains(vesselActivityCode);

-        boolean whenLeaving = WHEN_LEAVING_CODE.contains(vesselActivityCode);

+        // Is the floating object can be created by the observed systems?

+        boolean floatingObjectCreatedByObservedSystem = observedSystemCodes.stream().anyMatch(OBSERVED_SYTEM_CODES_WITH_DCP::contains);

+

+        // Is the current vessel activity accept to create Floating object?

+        boolean vesselActivityAllowFad = vesselActivity.isAllowFad();

+

+        // Get object type code

         String objectTypeCode = resultSet.getString(28);

         if (objectTypeCode == null) {

-            //FIXME should fail ?

+            // If null, then consider it as not found in avdth

             objectTypeCode = "999";

         }

-        //FIXME 29 - F_DCP_ECO

-//        int dcpEcoCode = resultSet.getInt(29);

-        String buoyOwnershipCode = resultSet.getString(30);

+

+        // Is floating object exists in avdth?

+        boolean floatingObjectExistsInAvdth = !objectTypeCode.equals("999");

+

+        if (!floatingObjectExistsInAvdth && !floatingObjectCreatedByObservedSystem) {

+            // If Floating object does not exist in avdth, neither need to be created by observed system,

+            // Do not create the floating object (no log is required here)

+            reset(dataContext);

+            return null;

+        }

+

+        if (!vesselActivityAllowFad) {

+            // If ObServe veseel activity does not support Floating object, reject import

+            List<Object> activityPrimaryKey = ActivityReader.getActivityPk(resultSet);

+            String activityPk = Query.primaryKeyString(activityPrimaryKey);

+            String message = String.format("On activity %s (ACTIVITE.C_OPERA %s), can not add Floating object since ObServe Vessel activity with code : %s does not allow FAD (floatingObjectExistsInAvdth? %b, floatingObjectCreatedByObservedSystem? %b).",

+                                           activityPk,

+                                           vesselActivityCode,

+                                           vesselActivity.getCode(),

+                                           floatingObjectExistsInAvdth,

+                                           floatingObjectCreatedByObservedSystem);

+            log.warn(message);

+            badVesselActivityCodes.computeIfAbsent(vesselActivityCode, MutableInt::new).increment();

+

+//            // do not create the floating object

+//            reset(dataContext);

+//            return null;

+        }

+

+        FloatingObject entity = newEntity(FloatingObject.SPI);

+

+        // Get floating object operation

+        ObjectOperation objectOperation = getObjectOperationByVesselActivityCode(dataContext, vesselActivityCode);

+        entity.setObjectOperation(Objects.requireNonNull(objectOperation, String.format("No mapping for FloatingObject.objectOperation from ACTIVITE.C_OPERA: %s", vesselActivityCode)));

+

         String buoyTypeCode = resultSet.getString(31);

         if (buoyTypeCode == null) {

-            //FIXME should fail?

+            // If null, then consider it as not found in avdth

             buoyTypeCode = "999";

         }

-        String buoyId = resultSet.getString(32);

-        String dcpWeight = resultSet.getString(33);

-        if (dcpWeight != null) {

-            //FIXME Check if ok

-            entity.setComment(String.format("ACTIVITE.POIDS_ESTIM_DCP value from avdth: %s", dcpWeight));

-        }

-        ObservedSystem observedSystem = dataContext.getObservedSystem(objectTypeCode, buoyTypeCode);

+        // IS buoy exists in avdth?

+        boolean buoyExistsInAvdth = !"999".equals(buoyTypeCode);

+

+        ObservedSystem observedSystem = getObservedSystem(dataContext, objectTypeCode, floatingObjectExistsInAvdth, buoyExistsInAvdth);

         if (observedSystem != null) {

-            dataContext.getActivity().addObservedSystem(observedSystem);

+            activity.addObservedSystem(observedSystem);

         }

-        // avdth data has a buoy

-        boolean willCreateBuoy = !"999".equals(buoyTypeCode);

-        // to add default material if there is a declared buoy and no object material created by ObservedSystem

+

+        boolean addBuoyFromObservedSystem = addFloatingObjectMaterials(dataContext,

+                                                                       resultSet,

+                                                                       vesselActivityCode,

+                                                                       objectTypeCode,

+                                                                       observedSystemCodes,

+                                                                       entity,

+                                                                       floatingObjectExistsInAvdth,

+                                                                       buoyExistsInAvdth);

+

+        TransmittingBuoy transmittingBuoy = null;

+        if (buoyExistsInAvdth) {

+            transmittingBuoy = getBuoyFromAvdthActivity(dataContext, resultSet, buoyTypeCode, vesselActivityCode, objectOperation, activity);

+        } else if (addBuoyFromObservedSystem) {

+            transmittingBuoy = getBuoyFromObservedSystem(dataContext);

+        }

+        if (transmittingBuoy != null) {

+            // only add buoy if buoy type known

+            entity.addTransmittingBuoy(transmittingBuoy);

+        }

+        reset(dataContext);

+        return entity;

+    }

+

+    private boolean addFloatingObjectMaterials(ImportDataContext dataContext,

+                                               ResultSet resultSet,

+                                               String vesselActivityCode,

+                                               String objectTypeCode,

+                                               Set<String> observedSystemCodes,

+                                               FloatingObject entity,

+                                               boolean floatingObjectExistsInAvdth,

+                                               boolean buoyExistsInAvdth) throws SQLException {

+        boolean whenArriving = WHEN_ARRIVING_CODE.contains(vesselActivityCode);

+        boolean whenLeaving = WHEN_LEAVING_CODE.contains(vesselActivityCode);

+

+        // to add default material if there is a declared buoy (but no floating object declared)

+        // and no object material created by ObservedSystem, we will then add a standalone buoy

         boolean addDefaultObjectMaterialOnUnknownObjectType = true;

         // should create a buoy from ObservedSystem

         boolean addBuoyFromObservedSystem = false;

         Set<String> objectMaterialUsed = new TreeSet<>();

+

+        if (floatingObjectExistsInAvdth) {

+            Object dcpEcoCode = resultSet.getObject(29);

+            if (dcpEcoCode != null && Objects.equals(2, dcpEcoCode)) {

+                // add material 4-1 (Biodegradable materials)

+                addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialBiodegradable(), whenArriving, whenLeaving, entity);

+            }

+        }

+

         if (observedSystemCodes.contains("20")) {

             // add ObjectMaterial FOB

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialFOB(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialFOB(), whenArriving, whenLeaving, entity);

             addDefaultObjectMaterialOnUnknownObjectType = false;

         } else if (observedSystemCodes.contains("21")) {

             // add ObjectMaterial 2-1 NLOG

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity);

             addDefaultObjectMaterialOnUnknownObjectType = false;

         } else if (observedSystemCodes.contains("22")) {

             // add ObjectMaterial 2-1 NLOG + 1-1 DFAD

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity);

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialNLOG(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity);

             addDefaultObjectMaterialOnUnknownObjectType = false;

             addBuoyFromObservedSystem = true;

         } else if (observedSystemCodes.contains("23")) {

             // add ObjectMaterial 2-2 ALOG

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity);

             addDefaultObjectMaterialOnUnknownObjectType = false;

         } else if (observedSystemCodes.contains("24")) {

             // add ObjectMaterial 2-2 ALOG + 1-1 DFAD

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity);

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialALOG(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialDFAD(), whenArriving, whenLeaving, entity);

             addDefaultObjectMaterialOnUnknownObjectType = false;

             addBuoyFromObservedSystem = true;

         } else if (observedSystemCodes.contains("25")) {

             // add ObjectMaterial 1-2 AFAD

-            addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialAFAD(), whenArriving, whenLeaving, entity);

+            addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialAFAD(), whenArriving, whenLeaving, entity);

             addDefaultObjectMaterialOnUnknownObjectType = false;

         }

-        if ("999".equals(objectTypeCode)) {

-            if (willCreateBuoy && addDefaultObjectMaterialOnUnknownObjectType) {

-                // add ObjectMaterial 2-2-4-4

-                addObjectMaterial(objectMaterialUsed, dataContext.getObjectMaterialAlone(), whenArriving, whenLeaving, entity);

+        if (floatingObjectExistsInAvdth) {

+            // simple mapping

+            ObjectMaterial objectMaterial = getObjectMaterialByObjectTypeCode(dataContext, objectTypeCode);

+            if (objectMaterial != null) {

+                addObjectMaterial(vesselActivityCode, objectMaterialUsed, objectMaterial, whenArriving, whenLeaving, entity);

             }

         } else {

-            // simple mapping

-            ObjectMaterial objectMaterial1 = dataContext.getObjectMaterialByObjectTypeCode(objectTypeCode);

-            if (objectMaterial1 != null) {

-                addObjectMaterial(objectMaterialUsed, objectMaterial1, whenArriving, whenLeaving, entity);

+            if (buoyExistsInAvdth && addDefaultObjectMaterialOnUnknownObjectType) {

+                // add ObjectMaterial 2-2-4-4

+                addObjectMaterial(vesselActivityCode, objectMaterialUsed, dataContext.getObjectMaterialAlone(), whenArriving, whenLeaving, entity);

             }

         }

-        if (willCreateBuoy) {

-            // only add buoy if buoy type known

-            TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyTypeByBuoyTypeCode(buoyTypeCode);

-            TransmittingBuoyOperation transmittingBuoyOperation = dataContext.getTransmittingBuoyOperationByVesselActivityCode(vesselActivityCode);

-            if (transmittingBuoyOperation != null) {

-                TransmittingBuoy transmittingBuoy = newEntity(TransmittingBuoy.SPI, transmittingBuoyCount);

-                transmittingBuoy.setTransmittingBuoyOperation(transmittingBuoyOperation);

-                transmittingBuoy.setTransmittingBuoyType(transmittingBuoyType);

-                transmittingBuoy.setCode(buoyId);

-                if (addCoordinate) {

-                    transmittingBuoy.setLatitude(dataContext.getActivity().getLatitude());

-                    transmittingBuoy.setLongitude(dataContext.getActivity().getLongitude());

-                }

-                TransmittingBuoyOwnership transmittingBuoyOwnership = dataContext.getTransmittingBuoyOwnership(buoyOwnershipCode);

-                transmittingBuoy.setTransmittingBuoyOwnership(transmittingBuoyOwnership);

-                entity.addTransmittingBuoy(transmittingBuoy);

-            }

-        } else if (addBuoyFromObservedSystem) {

-            // only add buoy if observed system ask for it (22 and 24)

-            TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyType98();

-            TransmittingBuoyOperation transmittingBuoyOperation = dataContext.getTransmittingBuoyOperation1();

+        return addBuoyFromObservedSystem;

+    }

+

+    private TransmittingBuoy getBuoyFromAvdthActivity(ImportDataContext dataContext, ResultSet resultSet, String buoyTypeCode, String vesselActivityCode, ObjectOperation objectOperation, Activity activity) throws SQLException {

+        boolean addCoordinate = "11".equals(objectOperation.getCode());

+        TransmittingBuoyType transmittingBuoyType = getTransmittingBuoyTypeByBuoyTypeCode(dataContext, buoyTypeCode);

+        TransmittingBuoyOperation transmittingBuoyOperation = getTransmittingBuoyOperationByVesselActivityCode(dataContext, vesselActivityCode);

+        if (transmittingBuoyOperation != null) {

+            String buoyOwnershipCode = resultSet.getString(30);

+            String buoyId = resultSet.getString(32);

             TransmittingBuoy transmittingBuoy = newEntity(TransmittingBuoy.SPI, transmittingBuoyCount);

             transmittingBuoy.setTransmittingBuoyOperation(transmittingBuoyOperation);

             transmittingBuoy.setTransmittingBuoyType(transmittingBuoyType);

-            transmittingBuoy.setCode(null);

-            entity.addTransmittingBuoy(transmittingBuoy);

+            transmittingBuoy.setCode(buoyId);

+            if (addCoordinate) {

+                transmittingBuoy.setLatitude(activity.getLatitude());

+                transmittingBuoy.setLongitude(activity.getLongitude());

+            }

+            TransmittingBuoyOwnership transmittingBuoyOwnership = getTransmittingBuoyOwnership(dataContext, buoyOwnershipCode);

+            transmittingBuoy.setTransmittingBuoyOwnership(transmittingBuoyOwnership);

+            return transmittingBuoy;

         }

-        reset(dataContext);

-        return entity;

+        return null;

+    }

+

+    private TransmittingBuoy getBuoyFromObservedSystem(ImportDataContext dataContext) {

+        // only add buoy if observed system ask for it (22 and 24)

+        TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyType98();

+        TransmittingBuoyOperation transmittingBuoyOperation = dataContext.getTransmittingBuoyOperation1();

+        TransmittingBuoy transmittingBuoy = newEntity(TransmittingBuoy.SPI, transmittingBuoyCount);

+        transmittingBuoy.setTransmittingBuoyOperation(transmittingBuoyOperation);

+        transmittingBuoy.setTransmittingBuoyType(transmittingBuoyType);

+        transmittingBuoy.setCode(null);

+        return transmittingBuoy;

     }

     private void reset(ImportDataContext dataContext) {

         dataContext.setObservedSystemCodes(null);

     }

-    private void addObjectMaterial(Set<String> objectMaterialUsed, ObjectMaterial objectMaterial, boolean whenArriving, boolean whenLeaving, FloatingObject floatingObject) {

+    private void addObjectMaterial(String vesselActivityCode, Set<String> objectMaterialUsed, ObjectMaterial objectMaterial, boolean whenArriving, boolean whenLeaving, FloatingObject floatingObject) {

         if (!whenArriving && !whenLeaving) {

-            throw new IllegalStateException("Can't have not whenArriving and not whenLeaving ");

+            throw new IllegalStateException(String.format("Can't have not whenArriving and not whenLeaving from ACTIVITE.C_OPERA: %s", vesselActivityCode));

         }

         if (!objectMaterialUsed.add(objectMaterial.getTopiaId())) {

             // already used

         return floatingObjectPartCount.intValue();

     }

+    public Map<String, MutableInt> getBadVesselActivityCodes() {

+        return badVesselActivityCodes;

+    }

+

+    private ObjectOperation getObjectOperationByVesselActivityCode(ImportDataContext dataContext, String vesselActivityCode) {

+        String objectOperationCode = DCP_OBJECT_OPERATION_CODE_MAPPING.get(vesselActivityCode);

+        return dataContext.getObjectOperation(Objects.requireNonNull(objectOperationCode, String.format("Can't find objectOperation with ACTIVITE.C_OPERA: %s", vesselActivityCode)));

+    }

+

+    private ObservedSystem getObservedSystem(ImportDataContext dataContext, String objectTypeCode, boolean floatingObjectExistsInAvdth,boolean buoyExistsInAvdth) {

+        if (!floatingObjectExistsInAvdth) {

+            if (!buoyExistsInAvdth) {

+                return dataContext.getObservedSystem0();

+            }

+            return dataContext.getObservedSystem20();

+        }

+        String observedSystemCode = OBSERVED_SYSTEM_CODE_MAPPING.get(objectTypeCode);

+        return dataContext.getObservedSystem(observedSystemCode, false);

+    }

+

+    private TransmittingBuoyOwnership getTransmittingBuoyOwnership(ImportDataContext dataContext, String buoyOwnershipCode) {

+        if (buoyOwnershipCode == null) {

+            return null;

+        }

+        String transmittingBuoyOwnershipCode = TRANSMITTING_BUOY_OWNERSHIP_CODE_MAPPING.get(buoyOwnershipCode);

+        if (transmittingBuoyOwnershipCode == null) {

+            return null;

+        }

+        return dataContext.getTransmittingBuoyOwnership(transmittingBuoyOwnershipCode);

+    }

+

+    private TransmittingBuoyOperation getTransmittingBuoyOperationByVesselActivityCode(ImportDataContext dataContext, String vesselActivityCode) {

+        String transmittingBuoyTypeCode = TRANSMITTING_BUOY_OPERATION_CODE_MAPPING.get(vesselActivityCode);

+        if (transmittingBuoyTypeCode == null) {

+            return null;

+        }

+        return dataContext.getTransmittingBuoyOperation(transmittingBuoyTypeCode);

+    }

+

+

+    private TransmittingBuoyType getTransmittingBuoyTypeByBuoyTypeCode(ImportDataContext dataContext, String buoyTypeCode) {

+        String transmittingBuoyTypeCode = TRANSMITTING_BUOY_TYPE_CODE_MAPPING.get(buoyTypeCode);

+        if (transmittingBuoyTypeCode == null) {

+            transmittingBuoyTypeCode = "999";

+        }

+        TransmittingBuoyType transmittingBuoyType = dataContext.getTransmittingBuoyType(transmittingBuoyTypeCode);

+        return Objects.requireNonNull(transmittingBuoyType, String.format("Can't find transmittingBuoyType with code: %s", buoyTypeCode));

+    }

+

+    /**

+     * To get objectMaterial associated by a DCP (or null) from avdth ACTIVITE.C_TYP_OBJET code.

+     *

+     * @param objectTypeCode ACTIVITE.C_TYP_OBJET

+     * @return material or null

+     */

+    private ObjectMaterial getObjectMaterialByObjectTypeCode(ImportDataContext dataContext, String objectTypeCode) {

+        switch (objectTypeCode) {

+            case "1":

+                return dataContext.getObjectMaterialAFAD();

+            case "2":

+                return dataContext.getObjectMaterialDFAD();

+            case "3":

+                return dataContext.getObjectMaterialLOG();

+            case "9":

+                return dataContext.getObjectMaterialFOB();

+            case "10":

+                return dataContext.getObjectMaterialFALOG();

+            case "11":

+                return dataContext.getObjectMaterialHALOG();

+            case "12":

+                return dataContext.getObjectMaterialANLOG();

+            case "13":

+                return dataContext.getObjectMaterialVNLOG();

+        }

+        return null;

+    }

 }
         super.toResult(resultBuilder);

         toResult(resultBuilder, FloatingObjectPart.class, getReader().getFloatingObjectPartCount());

         toResult(resultBuilder, TransmittingBuoy.class, getReader().getTransmittingBuoyCount());

+//        getReader().getBadVesselActivityCodes().forEach((k,v)-> {

+//            resultBuilder.put(String.format("Bad Vessel activity used to build FAD from ACTIVITY.C_OPERA %s used but ObServe not allow fad.", k), v.getValue());

+//        });

     }

 }

             if (!dataContext.isCanCreateActivity()) {

                 String message = String.format("Sample [%20s], not allowed to create missing activity and could not find Activity [%20s]", samplePk, activityPk);

                 log.debug(message);

-                context().addMessage(message);

+                addMessage(message);

                 return null;

             }

             // only on this case we can create missing route and activity (See #2070)

             String wellSetPk = Query.primaryKeyString(Query.getPrimaryKey(resultSet, 1, 3));

             String message = String.format("WellPlan [%20s], not allowed to create missing activity and could not find Activity [%20s]", wellSetPk, activityPk);

             log.debug(message);

-            context().addMessage(message);

+            addMessage(message);

             return null;

         } else {

             activity = new ActivityImpl();

   <description>ObServe Core Persistence Test module</description>

   <properties>

     <test_forceImport>false</test_forceImport>

+    <test_forcePrepare>false</test_forcePrepare>

     <test_allYears>false</test_allYears>

   </properties>

   <dependencies>

             <systemPropertyVariables>

               <test_forceImport>${test_forceImport}</test_forceImport>

               <test_allYears>${test_allYears}</test_allYears>

+              <test_forcePrepare>${test_forcePrepare}</test_forcePrepare>

             </systemPropertyVariables>

           </configuration>

         </plugin>

+package fr.ird.observe.persistence.avdth.data;

+

+/*-

+ * #%L

+ * ObServe Core :: Persistence :: Test

+ * %%

+ * Copyright (C) 2008 - 2023 IRD, Ultreia.io

+ * %%

+ * This program is free software: you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as

+ * published by the Free Software Foundation, either version 3 of the

+ * License, or (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public

+ * License along with this program.  If not, see

+ * <http://www.gnu.org/licenses/gpl-3.0.html>.

+ * #L%

+ */

+

+import fr.ird.observe.dto.ProtectedIdsCommon;

+import org.junit.runners.Parameterized;

+

+import java.nio.file.Path;

+

+/**

+ * Created on 19/01/2023.

+ *

+ * @author Tony Chemit - dev@tchemit.fr

+ * @since 9.0.24

+ */

+public class AvdthAtlanticDataBuilderByYearLatestTest extends AvdthDataBuilderByYearTestSupport {

+

+    public static final Path ROOT_PATH = getPath("OA").resolve("latest");

+

+    @Parameterized.Parameters(name = "{index}: {0}")

+    public static Iterable<?> data() {

+        return getDatabases(ROOT_PATH);

+    }

+

+    @Override

+    protected Path getRootPath() {

+        return ROOT_PATH;

+    }

+

+    @Override

+    protected String getOceanId() {

+        return ProtectedIdsCommon.COMMON_OCEAN_ATLANTIC;

+    }

+}

     public static List<String> getDatabases(Path path) {

         List<String> databases = allDatabases(path);

-        return AvdthFixtures.IMPORT_ALL_YEARS ? databases : databases.subList(0, 3);

+        return AvdthFixtures.IMPORT_ALL_YEARS || databases.size() < 3 ? databases : databases.subList(0, 3);

     }

     @Override

     }

     protected void checkNotValidResult(AvdthDataImportResult result) {

-        Assume.assumeTrue(result.getNotReadResult() + "", forceImport());

+        Map<String, Integer> notReadResult = result.getNotReadResult();

+        String message;

+        if (notReadResult.isEmpty()) {

+            message = result.getMessages() + "";

+        } else {

+            message = notReadResult + "";

+            return;

+        }

+        Assume.assumeTrue(message, forceImport());

     }

     protected long count() {

+package fr.ird.observe.persistence.avdth.data;

+

+/*-

+ * #%L

+ * ObServe Core :: Persistence :: Test

+ * %%

+ * Copyright (C) 2008 - 2023 IRD, Ultreia.io

+ * %%

+ * This program is free software: you can redistribute it and/or modify

+ * it under the terms of the GNU General Public License as

+ * published by the Free Software Foundation, either version 3 of the

+ * License, or (at your option) any later version.

+ *

+ * This program is distributed in the hope that it will be useful,

+ * but WITHOUT ANY WARRANTY; without even the implied warranty of

+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+ * GNU General Public License for more details.

+ *

+ * You should have received a copy of the GNU General Public

+ * License along with this program.  If not, see

+ * <http://www.gnu.org/licenses/gpl-3.0.html>.

+ * #L%

+ */

+

+import fr.ird.observe.dto.ProtectedIdsCommon;

+import org.junit.runners.Parameterized;

+

+import java.nio.file.Path;

+

+/**

+ * Created on 19/01/2023.

+ *

+ * @author Tony Chemit - dev@tchemit.fr

+ * @since 9.0.24

+ */

+public class AvdthIndianDataBuilderByYearLatestTest extends AvdthDataBuilderByYearTestSupport {

+

+    public static final Path ROOT_PATH = getPath("OI").resolve("latest");

+

+    @Parameterized.Parameters(name = "{index}: {0}")

+    public static Iterable<?> data() {

+        return getDatabases(ROOT_PATH);

+    }

+

+    @Override

+    protected Path getRootPath() {

+        return ROOT_PATH;

+    }

+

+    @Override

+    protected String getOceanId() {

+        return ProtectedIdsCommon.COMMON_OCEAN_INDIAN;

+    }

+}
     <!-- FIXME   <lib.version.jaxb-runtime>4.0.0</lib.version.jaxb-runtime>-->

     <lib.version.jts>1.16.1</lib.version.jts>

     <!-- FIXME   <lib.version.jts>1.18.2</lib.version.jts>-->

+    <lib.version.hsqldb>2.7.0</lib.version.hsqldb>

+    <!-- FIXME   <lib.version.hsqldb>2.7.1</lib.version.hsqldb>-->

     <!-- license header configuration -->

     <license.licenseName>gpl_v3</license.licenseName>

     <license.organizationName>IRD, Ultreia.io</license.organizationName>

 * Note 11

 ### <a name="n_0_11"></a>

-Donnée non persistée dans ObServe, on la conserve dans le commentaire du DCP

+Donnée non persistée dans ObServe, (voir https://gitlab.com/ultreiaio/ird-observe/-/issues/2575#note_1247253501).

 * Note 12

 ### <a name="n_0_12"></a>

Tony CHEMIT pushed to branch feature/issue_2575 at ultreiaio / ird-observe

Commits:

14 changed files:

Changes:

...	...	@@ -93,5 +93,9 @@ public abstract class DataReader<E extends DataEntity> {
93	93	protected final AvdthDataImportConfiguration getContext() {
94	94	return context.getConfiguration();
95	95	}
	96	+
	97	+ protected final void addMessage(String message) {
	98	+ context.addMessage(message);
	99	+ }
96	100	}
97	101

...	...	@@ -192,6 +192,9 @@ public class FloatingObjectWriter extends DataWriter<FloatingObject, FloatingObj
192	192	super.toResult(resultBuilder);
193	193	toResult(resultBuilder, FloatingObjectPart.class, getReader().getFloatingObjectPartCount());
194	194	toResult(resultBuilder, TransmittingBuoy.class, getReader().getTransmittingBuoyCount());
	195	+// getReader().getBadVesselActivityCodes().forEach((k,v)-> {
	196	+// resultBuilder.put(String.format("Bad Vessel activity used to build FAD from ACTIVITY.C_OPERA %s used but ObServe not allow fad.", k), v.getValue());
	197	+// });
195	198	}
196	199	}
197	200