Author: tchemit Date: 2011-12-12 14:22:57 +0100 (Mon, 12 Dec 2011) New Revision: 171 Url: http://forge.codelutin.com/repositories/revision/echobase/171 Log: clean acoustic data loaders Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataAcquisitionLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataProcessingLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransectLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransitLoader.java trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/VoyageLoader.java Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/CellLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -203,8 +203,8 @@ } protected void loadDatas() throws TopiaException { - List<DataProcessing> dataProcessings = getEntities(DataProcessing.class); - dataProcessingsById = Maps.uniqueIndex(dataProcessings, EchoBaseFunctions.DATA_PROCESSING_ID); + dataProcessingsById = getUniverse(DataProcessing.class, + EchoBaseFunctions.DATA_PROCESSING_ID); DataMetadataDAO dataMetadataDAO = EchoBaseDAOHelper.getDataMetadataDAO(getTransaction()); @@ -225,9 +225,8 @@ //TODO Fix name elementaryTimeMetadata = dataMetadataDAO.findByName("TimeEnd"); - List<DataQuality> dataQualities = getEntities(DataQuality.class); - dataQualitiesById = Maps.uniqueIndex(dataQualities, - EchoBaseFunctions.DATA_QUALITY_ID); + dataQualitiesById = getUniverse( + DataQuality.class, EchoBaseFunctions.DATA_QUALITY_ID); CellTypeDAO cellTypeDAO = EchoBaseDAOHelper.getCellTypeDAO( getTransaction()); @@ -270,7 +269,6 @@ Preconditions.checkNotNull(scenarioByProcessingId); Preconditions.checkNotNull(esduByProcessingId); Preconditions.checkNotNull(trancheDefById); - Preconditions.checkNotNull(trancheDefById); Preconditions.checkNotNull(eiConfigurationByProcessingId); dao = getDAO(); @@ -403,6 +401,8 @@ List<Cell> result = Lists.newArrayList(); + Collection<Map<String, Object>> esduRows = esduByProcessingId.get(processingId); + for (Map<String, Object> scenarioRow : scenarioByProcessingId.get(processingId)) { String rowStrateId = (String) scenarioRow.get("POST_STRATE_SCENARIO"); @@ -425,8 +425,7 @@ log.debug("Treat esdu " + cellStringId); } // keep this line - Collection<Map<String, Object>> esduRows = - esduByProcessingId.get(processingId); + for (Map<String, Object> esduRow : esduRows) { Date rowCellId = (Date) esduRow.get("DHTU_ESDU"); @@ -448,7 +447,7 @@ createEsduCellData(esduCell, cellId, - esduRows, + esduRow, acousticMetadata ); @@ -468,61 +467,52 @@ } private int createEsduCellData(Cell esduCell, Date esduCellId, - Collection<Map<String, Object>> esduRows, + Map<String, Object> esduRow, DataMetadata acousticMetadata ) throws TopiaException { DataQuality dataQuality = esduCell.getDataQuality(); - for (Map<String, Object> row : esduRows) { + Data latitudeData = dataDao.create( + Data.PROPERTY_DATA_NAME, "esduLatitude", + Data.PROPERTY_DATA_METADATA, esduLatitudeMetadata, + Data.PROPERTY_DATA_QUALITY, dataQuality, + Data.PROPERTY_DATA_VALUE, getFloatValue(esduRow, "LATDMC") + ); + esduCell.addData(latitudeData); - // find the correct row - Date rowCellId = (Date) row.get("DHTU_ESDU"); + Data longitudeData = dataDao.create( + Data.PROPERTY_DATA_NAME, "esduLongitude", + Data.PROPERTY_DATA_METADATA, esduLongitudeMetadata, + Data.PROPERTY_DATA_QUALITY, dataQuality, + Data.PROPERTY_DATA_VALUE, getFloatValue(esduRow, "LGDMC") + ); + esduCell.addData(longitudeData); - if (!esduCellId.equals(rowCellId)) { - continue; - } + Data depthData = dataDao.create( + Data.PROPERTY_DATA_NAME, "esduDepth", + Data.PROPERTY_DATA_METADATA, esduDepthMetadata, + Data.PROPERTY_DATA_QUALITY, dataQuality, + Data.PROPERTY_DATA_VALUE, getFloatValue(esduRow, "SONDE") + ); + esduCell.addData(depthData); - Data latitudeData = dataDao.create( - Data.PROPERTY_DATA_NAME, "esduLatitude", - Data.PROPERTY_DATA_METADATA, esduLatitudeMetadata, - Data.PROPERTY_DATA_QUALITY, dataQuality, - Data.PROPERTY_DATA_VALUE, getFloatValue(row, "LATDMC") - ); - esduCell.addData(latitudeData); + Data timeData = dataDao.create( + Data.PROPERTY_DATA_NAME, "esduTime", + Data.PROPERTY_DATA_METADATA, esduTimeMetadata, + Data.PROPERTY_DATA_QUALITY, dataQuality, + Data.PROPERTY_DATA_VALUE, esduCellId.toString() + ); + esduCell.addData(timeData); - Data longitudeData = dataDao.create( - Data.PROPERTY_DATA_NAME, "esduLongitude", - Data.PROPERTY_DATA_METADATA, esduLongitudeMetadata, - Data.PROPERTY_DATA_QUALITY, dataQuality, - Data.PROPERTY_DATA_VALUE, getFloatValue(row, "LGDMC") - ); - esduCell.addData(longitudeData); + Data acousticData = dataDao.create( + Data.PROPERTY_DATA_NAME, "esduAcoustic", + Data.PROPERTY_DATA_METADATA, acousticMetadata, + Data.PROPERTY_DATA_QUALITY, dataQuality, + Data.PROPERTY_DATA_VALUE, getFloatValue(esduRow, "TOTAL") + ); + esduCell.addData(acousticData); - Data depthData = dataDao.create( - Data.PROPERTY_DATA_NAME, "esduDepth", - Data.PROPERTY_DATA_METADATA, esduDepthMetadata, - Data.PROPERTY_DATA_QUALITY, dataQuality, - Data.PROPERTY_DATA_VALUE, getFloatValue(row, "SONDE") - ); - esduCell.addData(depthData); - - Data timeData = dataDao.create( - Data.PROPERTY_DATA_NAME, "esduTime", - Data.PROPERTY_DATA_METADATA, esduTimeMetadata, - Data.PROPERTY_DATA_QUALITY, dataQuality, - Data.PROPERTY_DATA_VALUE, esduCellId.toString() - ); - esduCell.addData(timeData); - - Data acousticData = dataDao.create( - Data.PROPERTY_DATA_NAME, "esduAcoustic", - Data.PROPERTY_DATA_METADATA, acousticMetadata, - Data.PROPERTY_DATA_QUALITY, dataQuality, - Data.PROPERTY_DATA_VALUE, getFloatValue(row, "TOTAL") - ); - esduCell.addData(acousticData); - } return esduCell.sizeData(); } @@ -638,55 +628,6 @@ return value.toString(); } -// private Map<DataProcessing, Map<String, Object>> getDataProcessings(Database db) throws IOException { -// Map<DataProcessing, Map<String, Object>> result = -// Maps.newLinkedHashMap(); -// -// Set<String> allProcessingIds = getAllAvailableProcessingIds(db); -// -// Set<String> noFoundIds = Sets.newHashSet(); -// -// for (Map<String, Object> row : db.getTable("EI_CONFIGURATION")) { -// String processingId = (String) row.get("ID_EI"); -// if (!allProcessingIds.contains(processingId)) { -// //TODO remove this test with new baracouda db -// noFoundIds.add(processingId); -// continue; -// } -// DataProcessing dataProcessing = -// dataProcessingsById.get(processingId.toLowerCase()); -// -// Map<String, Object> postStrateRow = findPostStrateRow( -// db, -// processingId -// ); -// -// result.put(dataProcessing, postStrateRow); -// -// if (postStrateRow == null) { -// if (log.isWarnEnabled()) { -// log.warn("Could not find postStrate with processing Id " + processingId); -// } -// } -// } -// -// for (String noFoundId : noFoundIds) { -// if (log.isWarnEnabled()) { -// log.warn("Could not find data processing data with id " + noFoundId); -// } -// } -// return result; -// } - -// private Set<String> getAllAvailableProcessingIds(Database db) throws IOException { -// Set<String> allProcessingIds = Sets.newHashSet(); -// for (Map<String, Object> row : db.getTable("SCENARIO")) { -// String id = row.get("ID_EI_SCENARIO").toString(); -// allProcessingIds.add(id); -// } -// return allProcessingIds; -// } - private Map<String, Object> findPostStrateRow(Database db, String processingId) throws IOException { Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataAcquisitionLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataAcquisitionLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataAcquisitionLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -62,14 +62,11 @@ @Override protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { - List<Voyage> voyages = getEntities(Voyage.class); - Map<String, Voyage> voyagesByName = Maps.uniqueIndex( - voyages, EchoBaseFunctions.VOYAGE_NAME); + Map<String, Voyage> voyagesByName = getUniverse( + Voyage.class, EchoBaseFunctions.VOYAGE_NAME); - List<AcousticInstrument> acousticInstruments = getEntities(AcousticInstrument.class); - Map<String, AcousticInstrument> acousticInstrumentsByName = - Maps.uniqueIndex(acousticInstruments, - EchoBaseFunctions.ACOUSTIC_INSTRUMENT_ID); + Map<String, AcousticInstrument> acousticInstrumentsByName = getUniverse( + AcousticInstrument.class, EchoBaseFunctions.ACOUSTIC_INSTRUMENT_ID); TopiaDAO<DataAcquisition> dao = getDAO(); @@ -81,15 +78,16 @@ for (Map<String, Object> row : db.getTable("EI_CONFIGURATION")) { String voyageName = (String) row.get("CAMPAGNE"); - Voyage voyage = voyagesByName.get(voyageName.toLowerCase()); + Voyage voyage = getFK(voyagesByName, voyageName); Transit transit = voyage.getTransit().iterator().next(); Transect transect = transit.getTransect().iterator().next(); String acousticInstrumentId = row.get("ID_REFSONDEUR").toString(); - AcousticInstrument acousticInstrument = - acousticInstrumentsByName.get(acousticInstrumentId.toLowerCase()); - Map<String, Object> sondeurRow = - EI_SONDEURByVesselName.get(acousticInstrumentId.toLowerCase()); + AcousticInstrument acousticInstrument = getFK( + acousticInstrumentsByName, acousticInstrumentId); + Map<String, Object> sondeurRow = getFK( + EI_SONDEURByVesselName, acousticInstrumentId); + Float transceiverAcquisitionAbsorption = (Float) sondeurRow.get("ALPHA"); Float transceiverAcquisitionPower = (Float) sondeurRow.get("PUISSANCE_MAX"); Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataProcessingLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataProcessingLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/DataProcessingLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -24,7 +24,6 @@ package fr.ifremer.echobase.tools.loaders; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.healthmarketscience.jackcess.Database; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.data.DataAcquisition; @@ -63,9 +62,8 @@ @Override protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { - List<Voyage> voyages = getEntities(Voyage.class); - Map<String, Voyage> voyagesByName = Maps.uniqueIndex( - voyages, EchoBaseFunctions.VOYAGE_NAME); + Map<String, Voyage> voyagesByName = getUniverse( + Voyage.class, EchoBaseFunctions.VOYAGE_NAME); TopiaDAO<DataProcessing> dao = getDAO(); @@ -73,12 +71,13 @@ for (Map<String, Object> row : db.getTable("EI_CONFIGURATION")) { String voyageName = (String) row.get("CAMPAGNE"); - Voyage voyage = voyagesByName.get(voyageName.toLowerCase()); + Voyage voyage = getFK(voyagesByName,voyageName); Transit transit = voyage.getTransit().iterator().next(); Transect transect = transit.getTransect().iterator().next(); // take the free DataAcquisition - Iterator<DataAcquisition> iterator = transect.getDataAcquisition().iterator(); + Iterator<DataAcquisition> iterator = + transect.getDataAcquisition().iterator(); DataAcquisition dataAcquisition = iterator.next(); while (!dataAcquisition.isDataProcessingEmpty()) { dataAcquisition = iterator.next(); Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/EntityLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -24,7 +24,9 @@ package fr.ifremer.echobase.tools.loaders; import com.google.common.base.Charsets; +import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.healthmarketscience.jackcess.Database; import fr.ifremer.echobase.EchoBaseTechnicalException; @@ -56,6 +58,8 @@ import java.io.InputStream; import java.io.Writer; import java.text.ParseException; +import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -382,4 +386,30 @@ getDbEditorService().exportDatas(associationMeta, out); } + + protected <E extends TopiaEntity,K> Map<K, E> getUniverse(Class<E> entityType, + Function<E, K> function) { + List<E> entities = getEntities(entityType); + Map<K, E> universe = Maps.uniqueIndex(entities, function); + return universe; + } + + protected <E> E getFK(Map<String, E> universe, String fk) { + return getFK(universe, fk, true, true); + } + + protected <E> E getFK(Map<String, E> universe, String fk, boolean lowerCase, boolean strict) { + E entity = universe.get(lowerCase?fk.toLowerCase():fk); + if (strict) + Preconditions.checkNotNull(entity, "Could not find entity with id " + fk); + else { + if (entity == null) { + if (log.isWarnEnabled()) { + log.warn("Could not find entity with id " + fk); + } + } + } + return entity; + } + } Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransectLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransectLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransectLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -24,7 +24,6 @@ package fr.ifremer.echobase.tools.loaders; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.healthmarketscience.jackcess.Database; import com.healthmarketscience.jackcess.Table; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; @@ -62,11 +61,11 @@ @Override protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { - List<Voyage> voyages = getEntities(Voyage.class); - Map<String, Voyage> voyagesByName = Maps.uniqueIndex(voyages, EchoBaseFunctions.VOYAGE_NAME); + Map<String, Voyage> voyagesByName = getUniverse( + Voyage.class, EchoBaseFunctions.VOYAGE_NAME); - List<Vessel> vessels = getEntities(Vessel.class); - Map<String, Vessel> vesselsByName = Maps.uniqueIndex(vessels, EchoBaseFunctions.VESSEL_NAME); + Map<String, Vessel> vesselsByName = getUniverse( + Vessel.class, EchoBaseFunctions.VESSEL_NAME); TopiaDAO<Transect> dao = getDAO(); @@ -75,9 +74,9 @@ for (Map<String, Object> row : table) { String vesselName = (String) row.get("NAVIRE"); - Vessel vessel = vesselsByName.get(vesselName.toLowerCase()); + Vessel vessel = getFK(vesselsByName, vesselName); String voyageName = (String) row.get("CAMPAGNE"); - Voyage voyage = voyagesByName.get(voyageName.toLowerCase()); + Voyage voyage = getFK(voyagesByName, voyageName); Transit transit = voyage.getTransit().iterator().next(); Transect transect = dao.create( Transect.PROPERTY_VESSEL, vessel, Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransitLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransitLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/TransitLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -24,7 +24,6 @@ package fr.ifremer.echobase.tools.loaders; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.healthmarketscience.jackcess.Database; import com.healthmarketscience.jackcess.Table; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; @@ -61,8 +60,8 @@ @Override protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { - List<Voyage> voyages = getEntities(Voyage.class); - Map<String, Voyage> voyagesByName = Maps.uniqueIndex(voyages, EchoBaseFunctions.VOYAGE_NAME); + Map<String, Voyage> voyagesByName = getUniverse( + Voyage.class, EchoBaseFunctions.VOYAGE_NAME); TopiaDAO<Transit> dao = getDAO(); @@ -71,7 +70,7 @@ for (Map<String, Object> row : table) { String voyageName = (String) row.get("CAMPAGNE"); - Voyage voyage = voyagesByName.get(voyageName.toLowerCase()); + Voyage voyage = getFK(voyagesByName, voyageName); Date startTime = voyage.getStartDate(); Date endTime = voyage.getEndDate(); Transit transit = dao.create( Modified: trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/VoyageLoader.java =================================================================== --- trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/VoyageLoader.java 2011-12-12 13:13:41 UTC (rev 170) +++ trunk/echobase-tools/src/main/java/fr/ifremer/echobase/tools/loaders/VoyageLoader.java 2011-12-12 13:22:57 UTC (rev 171) @@ -24,7 +24,6 @@ package fr.ifremer.echobase.tools.loaders; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.healthmarketscience.jackcess.Database; import com.healthmarketscience.jackcess.Table; import fr.ifremer.echobase.entities.data.Voyage; @@ -75,11 +74,12 @@ @Override protected void importBaracoudaEntities(Database db) throws IOException, TopiaException { - List<Mission> missions = getEntities(Mission.class); - Map<String, Mission> missionsByName = Maps.uniqueIndex(missions, EchoBaseFunctions.MISSION_NAME); + Map<String, Mission> missionsByName = getUniverse( + Mission.class, EchoBaseFunctions.MISSION_NAME); - List<AreaOfOperation> areaOfOperations = getEntities(AreaOfOperation.class); - Map<String, AreaOfOperation> areaOfOperationsByName = Maps.uniqueIndex(areaOfOperations, EchoBaseFunctions.AREA_OF_OPERATION_NAME); + Map<String, AreaOfOperation> areaOfOperationsByName = getUniverse( + AreaOfOperation.class, EchoBaseFunctions.AREA_OF_OPERATION_NAME); + TopiaDAO<Voyage> dao = getDAO(); List<Voyage> entities = Lists.newArrayList(); @@ -87,9 +87,10 @@ for (Map<String, Object> row : table) { String voyageName = (String) row.get("CAMPAGNE"); String missionName = (String) row.get("TYPE_CAMPAGNE"); - Mission mission = missionsByName.get(missionName.toLowerCase()); + Mission mission = getFK(missionsByName, missionName); String areaOfOperationName = (String) row.get("ZONE"); - AreaOfOperation areaOfOperation = areaOfOperationsByName.get(areaOfOperationName.toLowerCase()); + AreaOfOperation areaOfOperation = getFK(areaOfOperationsByName, + areaOfOperationName); Date startDate = (Date) row.get("DATE_DEPART"); Date endDate = (Date) row.get("DATE_ARRIVEE"); String startPort = (String) row.get("PORT_DEPART");