Author: tchemit Date: 2012-10-25 14:45:08 +0200 (Thu, 25 Oct 2012) New Revision: 743 Url: http://forge.codelutin.com/repositories/revision/echobase/743 Log: back to normal acoustic import Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-24 12:18:36 UTC (rev 742) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/AcousticImportService.java 2012-10-25 12:45:08 UTC (rev 743) @@ -25,9 +25,9 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; import fr.ifremer.echobase.EchoBaseFunctions; +import fr.ifremer.echobase.EchoBasePredicates; import fr.ifremer.echobase.csv.EchoBaseCsvUtil; import fr.ifremer.echobase.entities.EchoBaseEntityEnum; import fr.ifremer.echobase.entities.EchoBaseUser; @@ -50,6 +50,7 @@ import fr.ifremer.echobase.io.InputFile; import fr.ifremer.echobase.services.importdata.csv.AcousticImportModel; import fr.ifremer.echobase.services.importdata.csv.AcousticImportRow; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.nuiton.util.csv.Import; @@ -58,15 +59,14 @@ import java.io.Reader; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import static org.nuiton.i18n.I18n.l_; + /** * Service to launch a "acoustic data" import. * @@ -79,21 +79,19 @@ private static final Log log = LogFactory.getLog(AcousticImportService.class); - public static final Set<Integer> POSSIBLE_YEARS = Sets.newHashSet(); - @Override protected void startImport( AcousticImportConfiguration configuration, EchoBaseUser user) throws ImportException { -// // get selected voyage -// Voyage voyage = getEntityById(Voyage.class, -// configuration.getVoyageId()); -// -// // get selected vessel -// Vessel vessel = getEntityById(Vessel.class, -// configuration.getVesselId()); + // get selected voyage + Voyage voyage = getEntityById(Voyage.class, + configuration.getVoyageId()); + // get selected vessel + Vessel vessel = getEntityById(Vessel.class, + configuration.getVesselId()); + CellType esduCellType = getEntityByProperty( CellType.class, CellType.PROPERTY_ID, "Esdu"); Preconditions.checkNotNull(esduCellType); @@ -104,8 +102,8 @@ EchoBaseCsvFileImportResult importResult = importMoviesFile(configuration, - null,//voyage, - null,//vessel, + voyage, + vessel, esduCellType, elementaryCellType); configuration.addResult(importResult); @@ -137,24 +135,6 @@ return result; } - public Transit getTransit(List<Transit> transits, Date startTime) { - - Preconditions.checkNotNull(startTime); - Transit result = null; - for (Transit t : transits) { - Date transitEndTime = t.getEndTime(); - if (transitEndTime.before(startTime)) { - // transit before required range - continue; - } - - // ok transit contains required range - result = t; - break; - } - return result; - } - private EchoBaseCsvFileImportResult importMoviesFile(AcousticImportConfiguration configuration, Voyage voyage, Vessel vessel, @@ -182,23 +162,15 @@ AcousticImportModel csvModel = new AcousticImportModel( getCsvSeparator(), instrumentsById, dataQualityMap); -// DataAcquisitionDAO dataAcquisitionDAO = -// getDAO(DataAcquisition.class, DataAcquisitionDAO.class); -// DataProcessingDAO dataProcessingDAO = -// getDAO(DataProcessing.class, DataProcessingDAO.class); + DataAcquisitionDAO dataAcquisitionDAO = + getDAO(DataAcquisition.class, DataAcquisitionDAO.class); + DataProcessingDAO dataProcessingDAO = + getDAO(DataProcessing.class, DataProcessingDAO.class); CellDAO cellDAO = getDAO(Cell.class, CellDAO.class); DataDAO dataDAO = getDAO(Data.class, DataDAO.class); - List<Transit> allTransits = getEntities(Transit.class); - Collections.sort(allTransits, new Comparator<Transit>() { - @Override - public int compare(Transit o1, Transit o2) { - return o1.getStartTime().compareTo(o2.getStartTime()); - } - }); + boolean addDataAcquisition = configuration.isAddDataAcquisition(); -// boolean addDataAcquisition = configuration.isAddDataAcquisition(); - String suffix = configuration.getCellPositionReference().getMetadataNameSuffix(); @@ -207,11 +179,8 @@ Reader reader = getInputFileReader(inputFile); Locale locale = getLocale(); String dataProcessingId = null; - Calendar instance = Calendar.getInstance(); try { - Set<String> badTransectIds = Sets.newHashSet(); - Import<AcousticImportRow> importer = Import.newImport(csvModel, reader); DataAcquisition dataAcquisition = null; @@ -224,9 +193,6 @@ Set<String> createdDataAcquisitions = Sets.newHashSet(); - Map<Integer, Integer> years = Maps.newHashMap(); - - for (AcousticImportRow row : importer) { doFlushTransaction(++rowNumber, inputFile, configuration); @@ -240,170 +206,140 @@ } Date startDate = row.getCellDateStart(); - instance.setTime(startDate); - int year = instance.get(Calendar.YEAR); - if (!POSSIBLE_YEARS.contains(year)) { - continue; - } - Date endDate = row.getCellDateEnd(); // get transect to use + Transit transit = voyage.getTransit(startDate); + if (transit == null) { -// Transit transit = voyage.getTransit(startDate); -// if (transit == null) { -// -// // can not find correct transit -// throw new ImportException( -// l_(locale, "echobase.importError.transit.notfound", -// voyage.getName(), -// startDate, -// rowNumber)); -// } -// -// Transect transect = transit.getTransect(vessel); -// if (transect == null) { -// -// // can not find correct transect -// throw new ImportException( -// l_(locale, "echobase.importError.transect.notfound", -// voyage.getName(), -// startDate, -// vessel.getName(), -// rowNumber)); -// } -// -// -// AcousticInstrument instrument = row.getAcousticInstrument(); -// boolean isME70 = -// EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); -// -// if (dataAcquisition == null || -// !instrument.equals(dataAcquisition.getAcousticInstrument())) { -// -// // need to use another data acquisition -// -// if (log.isDebugEnabled()) { -// log.debug("[row " + rowNumber + -// "] New instrument to use (" + -// instrument.getId() + ")"); -// } -// -// if (CollectionUtils.isNotEmpty(elementaryCells)) { -// -// // means miss a esdu cell -// // can not find correct transect -// throw new ImportException( -// l_(locale, "echobase.importError.esduCell.notfound", -// voyage.getName(), -// vessel.getName(), -// instrument.getId(), -// rowNumber)); -// } -// -// String softwareVersion = -// getSoftwareVersion(configuration, isME70); -// -// String soundSpeedCalculations = -// getSoundSpeedCalculations(configuration, isME70); -// -// // try to get existing data acquisition -// dataAcquisition = getDataAcquisition( -// transect, -// instrument, -// addDataAcquisition -// ); -// -// if (dataAcquisition == null) { -// -// // need to create the data acquisition -// dataAcquisition = createDataAcquisition( -// configuration, -// instrument, -// softwareVersion, -// soundSpeedCalculations, -// row, -// dataAcquisitionDAO -// ); -// -// // collect id of the import -// importResult.addId(EchoBaseEntityEnum.DataAcquisition, -// dataAcquisition); -// createdDataAcquisitions.add( -// dataAcquisition.getTopiaId()); -// -// // add dataAcquisition to transect -// transect.addDataAcquisition(dataAcquisition); -// -// if (log.isDebugEnabled()) { -// log.debug("[row " + rowNumber + -// "] New dataAquisition to use (number: " + -// importResult.getNumberCreated( -// EchoBaseEntityEnum.DataAcquisition) + ")"); -// } -// -// // create data processing -// dataProcessing = createDataProcessing( -// configuration, -// dataProcessingId, -// softwareVersion, -// soundSpeedCalculations, -// row, -// dataProcessingDAO -// ); -// -// if (log.isDebugEnabled()) { -// log.debug("[row " + rowNumber + -// "] New dataProcessing to use (" + -// dataProcessing.getId() + ")"); -// } -// -// importResult.incrementsNumberCreated( -// EchoBaseEntityEnum.DataProcessing); -// -// // add it to data acquisition -// dataAcquisition.addDataProcessing(dataProcessing); -// } -// } -// -// if (!row.isCellToAdd()) { -// -// // dead cell not to be imported -// if (log.isDebugEnabled()) { -// log.debug("Will not import cell of row " + rowNumber); -// } -// continue; -// } + // can not find correct transit + throw new ImportException( + l_(locale, "echobase.importError.transit.notfound", + voyage.getName(), + startDate, + rowNumber)); + } - int cellType = row.getCellType(); + Transect transect = transit.getTransect(vessel); + if (transect == null) { - String esduCellId = cellDateFormat.format(endDate); + // can not find correct transect + throw new ImportException( + l_(locale, "echobase.importError.transect.notfound", + voyage.getName(), + startDate, + vessel.getName(), + rowNumber)); + } - DataQuality dataQuality = row.getDataQuality(); + AcousticInstrument instrument = row.getAcousticInstrument(); + boolean isME70 = + EchoBasePredicates.IS_ACOUSTIC_INSTRUMENT_ME70.apply(instrument); - Transit transit = getTransit(allTransits, startDate); - Transect transect = transit.getTransect().iterator().next(); - if (transect.isDataAcquisitionEmpty()) { + if (dataAcquisition == null || + !instrument.equals(dataAcquisition.getAcousticInstrument())) { - // no dataAcquisition - if (!badTransectIds.contains(transect.getTopiaId())) { - badTransectIds.add(transect.getTopiaId()); - if (log.isErrorEnabled()) { - log.error("Could not find dataAcquision for transit " + transit.getStartTime() + " - " + transect.getTitle()); + // need to use another data acquisition + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + + "] New instrument to use (" + + instrument.getId() + ")"); + } + + if (CollectionUtils.isNotEmpty(elementaryCells)) { + + // means miss a esdu cell + // can not find correct transect + throw new ImportException( + l_(locale, "echobase.importError.esduCell.notfound", + voyage.getName(), + vessel.getName(), + instrument.getId(), + rowNumber)); + } + + String softwareVersion = + getSoftwareVersion(configuration, isME70); + + String soundSpeedCalculations = + getSoundSpeedCalculations(configuration, isME70); + + // try to get existing data acquisition + dataAcquisition = getDataAcquisition( + transect, + instrument, + addDataAcquisition + ); + + if (dataAcquisition == null) { + + // need to create the data acquisition + dataAcquisition = createDataAcquisition( + configuration, + instrument, + softwareVersion, + soundSpeedCalculations, + row, + dataAcquisitionDAO + ); + + // collect id of the import + importResult.addId(EchoBaseEntityEnum.DataAcquisition, + dataAcquisition); + createdDataAcquisitions.add( + dataAcquisition.getTopiaId()); + + // add dataAcquisition to transect + transect.addDataAcquisition(dataAcquisition); + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + + "] New dataAquisition to use (number: " + + importResult.getNumberCreated( + EchoBaseEntityEnum.DataAcquisition) + ")"); } + + // create data processing + dataProcessing = createDataProcessing( + configuration, + dataProcessingId, + softwareVersion, + soundSpeedCalculations, + row, + dataProcessingDAO + ); + + if (log.isDebugEnabled()) { + log.debug("[row " + rowNumber + + "] New dataProcessing to use (" + + dataProcessing.getId() + ")"); + } + + importResult.incrementsNumberCreated( + EchoBaseEntityEnum.DataProcessing); + + // add it to data acquisition + dataAcquisition.addDataProcessing(dataProcessing); } + } + + if (!row.isCellToAdd()) { + + // dead cell not to be imported + if (log.isDebugEnabled()) { + log.debug("Will not import cell of row " + rowNumber); + } continue; } - Integer nb = years.get(year); - if (nb == null) { - nb = 0; - } - nb++; - years.put(year, nb); - dataAcquisition = transect.getDataAcquisition().iterator().next(); - dataProcessing = dataAcquisition.getDataProcessing().iterator().next(); + int cellType = row.getCellType(); + String esduCellId = cellDateFormat.format(endDate); + + DataQuality dataQuality = row.getDataQuality(); + + // collect Esdu cellIds only if dataAcquisitions was not created here boolean collectCellIds = !createdDataAcquisitions.contains( dataAcquisition.getTopiaId()); @@ -490,19 +426,6 @@ } } - List<Integer> sortedYears = Lists.newArrayList(years.keySet()); - Collections.sort(sortedYears); - for (Integer year : sortedYears) { - if (log.isInfoEnabled()) { - log.info("Seens years " + year + " :: " + years.get(year)); - } - } - for (String badTransectId : badTransectIds) { - if (log.isInfoEnabled()) { - log.info("Missing dataAcquisition for transect " + badTransectId); - } - } - return importResult; } catch (ImportRuntimeException e) { throw new ImportException(locale, inputFile, e); @@ -533,10 +456,8 @@ // create Depth data dataValue = row.getEsduCellDataDepth(); - if (dataValue != null) { createCellData(dao, cell, dataMetadatas, "DepthRefSurface" + suffix, dataValue, dataQuality, importResult); - } // create Time Start data dataValue = cellDateFormat.format(row.getCellDateStart()); Modified: trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java =================================================================== --- trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-24 12:18:36 UTC (rev 742) +++ trunk/echobase-services/src/main/java/fr/ifremer/echobase/services/importdata/csv/AcousticImportModel.java 2012-10-25 12:45:08 UTC (rev 743) @@ -62,13 +62,13 @@ newIgnoredColumn("MOVIES_EILayer\\sndset\\mainBeamAthwartSteeringAngle");//L newMandatoryColumn("MOVIES_EILayer\\sndset\\absorptionCoef", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_ABSORPTION, EchoBaseCsvUtil.FLOAT);//M newMandatoryColumn("MOVIES_EILayer\\sndset\\transmissionPower", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_POWER, EchoBaseCsvUtil.INTEGER);//N - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//O - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//P + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAlongAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ALONGSHIP, EchoBaseCsvUtil.FLOAT);//O + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamAthwartAngleSensitivity", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_BEAM_ANGLE_ATHWARTSHIP, EchoBaseCsvUtil.FLOAT);//P newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAlong");//Q newIgnoredColumn("MOVIES_EILayer\\sndset\\beam3dBWidthAthwart");//R - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//S - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//T - newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.NA_TO_PRIMITIVE_FLOAT_PARSER);//U + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamEquTwoWayAngle", AcousticImportRow.PROPERTY_TRANSDUCER_ACQUISITION_PSI, EchoBaseCsvUtil.FLOAT);//S + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamGain", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_GAIN, EchoBaseCsvUtil.FLOAT);//T + newMandatoryColumn("MOVIES_EILayer\\sndset\\beamSACorrection", AcousticImportRow.PROPERTY_TRANSCEIVER_ACQUISITION_SACORRECTION, EchoBaseCsvUtil.FLOAT);//U newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinDepth");//V newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMaxDepth");//W newIgnoredColumn("MOVIES_EILayer\\sndset\\bottomDetectionMinLevel");//X @@ -94,7 +94,7 @@ newIgnoredColumn("MOVIES_EILayer\\shipnav\\roll");//AR newIgnoredColumn("MOVIES_EILayer\\shipnav\\pitch");//AS newIgnoredColumn("MOVIES_EILayer\\shipnav\\heave");//AT - //newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU + newMandatoryColumn("MOVIES_EILayer\\shipnav\\depth", AcousticImportRow.PROPERTY_ESDU_CELL_DATA_DEPTH);//AU newIgnoredColumn("MOVIES_EILayer\\shipnav\\draught");//AV // Cell elementary Datas @@ -105,8 +105,8 @@ newMandatoryColumn("MOVIES_EILayer\\cellset\\depthend", AcousticImportRow.PROPERTY_CELL_DEPTH_END, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BA newIgnoredColumn("MOVIES_EILayer\\cellset\\indexstart");//BB newIgnoredColumn("MOVIES_EILayer\\cellset\\indexend");//BC - newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BD - newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE2);//BE + newMandatoryColumn("MOVIES_EILayer\\cellset\\datestart", AcousticImportRow.PROPERTY_CELL_DATE_START, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BD + newMandatoryColumn("MOVIES_EILayer\\cellset\\dateend", AcousticImportRow.PROPERTY_CELL_DATE_END, EchoBaseCsvUtil.IMPORT_DAY_TIME_ECHOBASE);//BE newMandatoryColumn("MOVIES_EILayer\\cellset\\lat", AcousticImportRow.PROPERTY_CELL_LATITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BF newMandatoryColumn("MOVIES_EILayer\\cellset\\long", AcousticImportRow.PROPERTY_CELL_LONGITUDE, EchoBaseCsvUtil.PRIMITIVE_FLOAT);//BG @@ -119,8 +119,8 @@ newIgnoredColumn("MOVIES_EILayer\\eilayer");//BN newMandatoryColumn("MOVIES_EILayer\\eilayer\\sa", AcousticImportRow.PROPERTY_CELL_NASC, EchoBaseCsvUtil.NA_TO_FLOAT_PARSER);//BO newIgnoredColumn("MOVIES_EILayer\\eilayer\\sv");//BP - //newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ - //newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR + newMandatoryColumn("MOVIES_EILayer\\eilayer\\ni", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_ECHO_INTEGRATED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BQ + newMandatoryColumn("MOVIES_EILayer\\eilayer\\nt", AcousticImportRow.PROPERTY_CELL_NUMBER_OF_SAMPLES_RECORDED, EchoBaseCsvUtil.NA_TO_INTEGER_PARSER);//BR newIgnoredColumn("MOVIES_EILayer\\boterr");//BR newIgnoredColumn("MOVIES_EILayer\\boterr\\sa");//BT newIgnoredColumn("MOVIES_EILayer\\boterr\\ni");//BU