[tutti] branch feature/importBigfin updated (44e4500 -> 483e0de)
This is an automated email from the git hooks/post-receive script. New change to branch feature/importBigfin in repository tutti. See http://git.codelutin.com/tutti.git from 44e4500 refs #5411 [CAPTURE] Import BIGFIN new 483e0de refs #5411 [CAPTURE] Import BIGFIN The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "adds" were already present in the repository and have only been added to this reference. Detailed log of new commits: commit 483e0dedd4bc38bd18cbdd2af8de24cdd4db2bd6 Author: Kevin Morin <morin@codelutin.com> Date: Tue Sep 9 07:57:22 2014 +0200 refs #5411 [CAPTURE] Import BIGFIN Summary of changes: .../persistence/entities/data/SpeciesBatchs.java | 21 ++ .../i18n/tutti-persistence_en_GB.properties | 1 + .../i18n/tutti-persistence_fr_FR.properties | 1 + .../tutti/service/bigfin/BigfinImportResult.java | 32 +- .../tutti/service/bigfin/BigfinImportService.java | 405 ++++++++++++--------- .../fr/ifremer/tutti/service/bigfin/Signs.java | 44 +++ .../resources/i18n/tutti-service_en_GB.properties | 5 + .../resources/i18n/tutti-service_fr_FR.properties | 4 + .../service/bigfin/BigfinImportServiceTest.java | 91 +++-- .../tutti/ui/swing/action/ImportBigfinAction.java | 25 +- .../resources/i18n/tutti-ui-swing_en_GB.properties | 5 + .../resources/i18n/tutti-ui-swing_fr_FR.properties | 10 +- 12 files changed, 405 insertions(+), 239 deletions(-) create mode 100644 tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/entities/data/SpeciesBatchs.java -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
This is an automated email from the git hooks/post-receive script. New commit to branch feature/importBigfin in repository tutti. See http://git.codelutin.com/tutti.git commit 483e0dedd4bc38bd18cbdd2af8de24cdd4db2bd6 Author: Kevin Morin <morin@codelutin.com> Date: Tue Sep 9 07:57:22 2014 +0200 refs #5411 [CAPTURE] Import BIGFIN --- .../persistence/entities/data/SpeciesBatchs.java | 21 ++ .../i18n/tutti-persistence_en_GB.properties | 1 + .../i18n/tutti-persistence_fr_FR.properties | 1 + .../tutti/service/bigfin/BigfinImportResult.java | 32 +- .../tutti/service/bigfin/BigfinImportService.java | 405 ++++++++++++--------- .../fr/ifremer/tutti/service/bigfin/Signs.java | 44 +++ .../resources/i18n/tutti-service_en_GB.properties | 5 + .../resources/i18n/tutti-service_fr_FR.properties | 4 + .../service/bigfin/BigfinImportServiceTest.java | 91 +++-- .../tutti/ui/swing/action/ImportBigfinAction.java | 25 +- .../resources/i18n/tutti-ui-swing_en_GB.properties | 5 + .../resources/i18n/tutti-ui-swing_fr_FR.properties | 10 +- 12 files changed, 405 insertions(+), 239 deletions(-) diff --git a/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/entities/data/SpeciesBatchs.java b/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/entities/data/SpeciesBatchs.java new file mode 100644 index 0000000..3c8741e --- /dev/null +++ b/tutti-persistence/src/main/java/fr/ifremer/tutti/persistence/entities/data/SpeciesBatchs.java @@ -0,0 +1,21 @@ +package fr.ifremer.tutti.persistence.entities.data; + +import com.google.common.base.Function; + +import java.io.Serializable; + +/** + * @author Kevin Morin (Code Lutin) + * @since x.x + */ +public class SpeciesBatchs extends AbstractSpeciesBatchs { + + public static final Function<SpeciesBatch, Serializable> GET_SAMPLE_CATEGORY_VALUE = + new Function<SpeciesBatch, Serializable>() { + @Override + public Serializable apply(SpeciesBatch input) { + return input.getSampleCategoryValue(); + } + }; + +} diff --git a/tutti-persistence/src/main/resources/i18n/tutti-persistence_en_GB.properties b/tutti-persistence/src/main/resources/i18n/tutti-persistence_en_GB.properties index 638a1f8..04c9353 100644 --- a/tutti-persistence/src/main/resources/i18n/tutti-persistence_en_GB.properties +++ b/tutti-persistence/src/main/resources/i18n/tutti-persistence_en_GB.properties @@ -37,6 +37,7 @@ adagio.enumeration.QualitativeValueId.MATURITY_2.description= adagio.enumeration.QualitativeValueId.MATURITY_3.description= adagio.enumeration.QualitativeValueId.MATURITY_4.description= adagio.enumeration.QualitativeValueId.MATURITY_5.description= +adagio.enumeration.QualitativeValueId.NOT_SIZED.description= adagio.enumeration.QualitativeValueId.SEX_UNDEFINED.description= application.common.unit=Unit application.common.unit.g=Gram diff --git a/tutti-persistence/src/main/resources/i18n/tutti-persistence_fr_FR.properties b/tutti-persistence/src/main/resources/i18n/tutti-persistence_fr_FR.properties index b73f810..b47ff7d 100644 --- a/tutti-persistence/src/main/resources/i18n/tutti-persistence_fr_FR.properties +++ b/tutti-persistence/src/main/resources/i18n/tutti-persistence_fr_FR.properties @@ -37,6 +37,7 @@ adagio.enumeration.QualitativeValueId.MATURITY_2.description= adagio.enumeration.QualitativeValueId.MATURITY_3.description= adagio.enumeration.QualitativeValueId.MATURITY_4.description= adagio.enumeration.QualitativeValueId.MATURITY_5.description= +adagio.enumeration.QualitativeValueId.NOT_SIZED.description= adagio.enumeration.QualitativeValueId.SEX_UNDEFINED.description= application.common.unit=Unité application.common.unit.g=Gramme diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java index 6478e79..407f10c 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportResult.java @@ -38,11 +38,13 @@ public class BigfinImportResult { protected final File importFile; + protected final List<String> fatalErrors = new ArrayList<>(); + protected final List<String> errors = new ArrayList<>(); - protected int nbVracImported; + protected int nbFrequenciesImported; - protected int nbHorsVracImported; + protected int nbFrequenciesDeleted; protected final List<Species> speciesNotInProtocol = new ArrayList<>(); @@ -54,24 +56,32 @@ public class BigfinImportResult { return importFile; } - public int getNbVracImported() { - return nbVracImported; + public int getNbFrequenciesImported() { + return nbFrequenciesImported; + } + + public int getNbFrequenciesDeleted() { + return nbFrequenciesDeleted; } - public int getNbHorsVracImported() { - return nbHorsVracImported; + public List<String> getFatalErrors() { + return fatalErrors; } public List<String> getErrors() { return errors; } - void incrementNbSortedImported() { - this.nbVracImported++; + void incrementNbFrequenciesImported(int nb) { + this.nbFrequenciesImported += nb; + } + + void incrementNbFrequenciesDeleted(int nb) { + this.nbFrequenciesDeleted += nb; } - void incrementNbUnsortedImported() { - this.nbHorsVracImported++; + void addFatalError(String error) { + fatalErrors.add(error); } void addError(String error) { @@ -83,6 +93,6 @@ public class BigfinImportResult { } public boolean isDone() { - return errors.isEmpty(); + return fatalErrors.isEmpty(); } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java index a8f7683..1f40312 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/BigfinImportService.java @@ -4,44 +4,34 @@ import com.google.common.base.Charsets; import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; -import com.google.common.collect.Sets; import com.google.common.io.Files; import fr.ifremer.adagio.core.dao.referential.pmfm.ObjectTypeCode2; -import fr.ifremer.adagio.core.dao.referential.pmfm.Pmfm; import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId; -import fr.ifremer.adagio.core.dao.referential.pmfm.PmfmId2; -import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValueId; -import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValueId2; -import fr.ifremer.tutti.persistence.entities.TuttiEntities; +import fr.ifremer.adagio.core.dao.referential.pmfm.QualitativeValue; import fr.ifremer.tutti.persistence.entities.data.Attachment; import fr.ifremer.tutti.persistence.entities.data.Attachments; import fr.ifremer.tutti.persistence.entities.data.BatchContainer; import fr.ifremer.tutti.persistence.entities.data.CatchBatch; import fr.ifremer.tutti.persistence.entities.data.FishingOperation; +import fr.ifremer.tutti.persistence.entities.data.SampleCategoryModel; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatch; -import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchBean; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequency; -import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequencyBean; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchFrequencys; import fr.ifremer.tutti.persistence.entities.data.SpeciesBatchs; import fr.ifremer.tutti.persistence.entities.protocol.SpeciesProtocol; import fr.ifremer.tutti.persistence.entities.protocol.TuttiProtocol; import fr.ifremer.tutti.persistence.entities.referential.Caracteristic; import fr.ifremer.tutti.persistence.entities.referential.CaracteristicQualitativeValue; -import fr.ifremer.tutti.persistence.entities.referential.CaracteristicQualitativeValues; import fr.ifremer.tutti.persistence.entities.referential.Species; import fr.ifremer.tutti.service.AbstractTuttiService; -import fr.ifremer.tutti.service.DecoratorService; import fr.ifremer.tutti.service.PersistenceService; import fr.ifremer.tutti.service.TuttiServiceContext; -import fr.ifremer.tutti.service.psionimport.PsionImportResult; import fr.ifremer.tutti.util.Weights; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateUtils; @@ -50,7 +40,6 @@ import org.apache.commons.logging.LogFactory; import org.nuiton.csv.Import; import org.nuiton.csv.ImportRuntimeException; import org.nuiton.jaxx.application.ApplicationBusinessException; -import org.nuiton.util.DateUtil; import java.io.File; import java.io.Reader; @@ -77,6 +66,9 @@ public class BigfinImportService extends AbstractTuttiService { protected PersistenceService persistenceService; + protected Caracteristic sizeCaracteristic; + protected Caracteristic genderCaracteristic; + protected Map<Signs, CaracteristicQualitativeValue> signsToCaracteristicValue; protected Map<String, SpeciesProtocol> speciesProtocolBySurveyCode; @@ -96,17 +88,17 @@ public class BigfinImportService extends AbstractTuttiService { } { // size caracteristic - Caracteristic caracteristic = persistenceService.getSizeCategoryCaracteristic(); - Signs.NOT_SIZED.registerSign(caracteristic, signsToCaracteristicValue); - Signs.SMALL.registerSign(caracteristic, signsToCaracteristicValue); - Signs.BIG.registerSign(caracteristic, signsToCaracteristicValue); + sizeCaracteristic = persistenceService.getSizeCategoryCaracteristic(); + Signs.NOT_SIZED.registerSign(sizeCaracteristic, signsToCaracteristicValue); + Signs.SMALL.registerSign(sizeCaracteristic, signsToCaracteristicValue); + Signs.BIG.registerSign(sizeCaracteristic, signsToCaracteristicValue); } { // sex caracteristic - Caracteristic caracteristic = persistenceService.getSexCaracteristic(); - Signs.NOT_SEXED.registerSign(caracteristic, signsToCaracteristicValue); - Signs.MALE.registerSign(caracteristic, signsToCaracteristicValue); - Signs.FEMALE.registerSign(caracteristic, signsToCaracteristicValue); + genderCaracteristic = persistenceService.getSexCaracteristic(); + Signs.NOT_SEXED.registerSign(genderCaracteristic, signsToCaracteristicValue); + Signs.MALE.registerSign(genderCaracteristic, signsToCaracteristicValue); + Signs.FEMALE.registerSign(genderCaracteristic, signsToCaracteristicValue); } } @@ -181,7 +173,7 @@ public class BigfinImportService extends AbstractTuttiService { if (log.isWarnEnabled()) { log.warn(error); } - result.addError(error); + result.addFatalError(error); } if (species == null || species.getId() == null) { // bloquer tout si un "species" ne match pas le référentiel de Tutti : lister dans ce cas les codes non reconnus @@ -190,7 +182,7 @@ public class BigfinImportService extends AbstractTuttiService { if (log.isWarnEnabled()) { log.warn(error); } - result.addError(error); + result.addFatalError(error); } } else { @@ -209,11 +201,15 @@ public class BigfinImportService extends AbstractTuttiService { } else if (speciesProtocol.getLengthStepPmfmId() == null && speciesInProtocolButWithoutLengthStepPmfmId.add(species)) { // bloquer toute espèce reconnue du protocole mais qui n'a pas de méthode de mesure - String error = t("tutti.service.bigfinImport.error.species.without.lengthstep", code); + String speciesLabel = species.getSurveyCode(); + if (StringUtils.isBlank(speciesLabel)) { + speciesLabel = species.getRefTaxCode(); + } + String error = t("tutti.service.bigfinImport.error.species.without.lengthstep", speciesLabel); if (log.isWarnEnabled()) { log.warn(error); } - result.addError(error); + result.addFatalError(error); } else { rows.add(bean); @@ -264,7 +260,7 @@ public class BigfinImportService extends AbstractTuttiService { SpeciesProtocol speciesProtocol = speciesProtocolBySurveyCode.get(code); Caracteristic lengthStepPmfm = persistenceService.getCaracteristic(Integer.parseInt(speciesProtocol.getLengthStepPmfmId())); - // get the rows whith the current species and separate them by vrac/hors varc + // get the rows with the current species and separate them by vrac/hors varc Collection<BigfinDataRow> speciesRows = rowsBySpecies.get(species); Multimap<Signs, BigfinDataRow> rowsByVracHorsVrac = Multimaps.index(speciesRows, new Function<BigfinDataRow, Signs>() { @@ -275,117 +271,49 @@ public class BigfinImportService extends AbstractTuttiService { } }); - //TODO kmorin 20140905 get categories from the conf - Test[] tests = new Test[] { - new Test(PmfmId.SORTED_UNSORTED, - new Function<BigfinDataRow, Signs>() { - @Override - public Signs apply(BigfinDataRow bigfinDataRow) { - Signs result = bigfinDataRow.getSzClass(); - return result; - } - }), - new Test(PmfmId.SIZE_CATEGORY, - new Function<BigfinDataRow, Signs>() { - @Override - public Signs apply(BigfinDataRow bigfinDataRow) { - Signs result = bigfinDataRow.getGender(); - return result; - } - }), - new Test(PmfmId.SEX, null) - }; - - test(operation, species, lengthStepPmfm, null, tests, 0, rowsByVracHorsVrac); -// // get the batches whose species is the current species and separate them by vrac/hors vrac -// Collection<SpeciesBatch> speciesSpeciesBatches = batchesBySpecies.get(species); -// Map<Serializable, SpeciesBatch> batchesByVracHorsVrac = Maps.uniqueIndex(speciesSpeciesBatches, -// new Function<SpeciesBatch, Serializable>() { -// @Override -// public Serializable apply(SpeciesBatch input) { -// return input.getSampleCategoryValue(); -// } -// }); -// -// // for each imported vrac/hors vrac found for the current species -// for (Signs vracHorsVrac : rowsByVracHorsVrac.keySet()) { -// -// // get the batch with the current vrac/hors vrac value -// SpeciesBatch batch = batchesByVracHorsVrac.get(signsToCaracteristicValue.get(vracHorsVrac)); -// // if it does not exists, create the batch -// if (batch == null) { -// batch = createSpeciesBatch(species, -// operation, -// PmfmId.SORTED_UNSORTED.getValue(), -// vracHorsVrac, -// null); -// } else { -// persistenceService.saveSpeciesBatchFrequency(batch.getId(), new ArrayList<SpeciesBatchFrequency>()); -// } -// -// // new vrac or hors vrac batch imported, increment the number -// if (Signs.VRAC.getQualitativeValueId().equals(vracHorsVrac.getQualitativeValueId())) { -// result.incrementNbSortedImported(); -// } else { -// result.incrementNbUnsortedImported(); -// } -// -// // get the imported rows with the current species and vrac / hors vrac and separate them by size -// Collection<BigfinDataRow> vracHorsVracRows = rowsByVracHorsVrac.get(vracHorsVrac); -// Multimap<Signs, BigfinDataRow> rowsBySize = -// Multimaps.index(vracHorsVracRows, new Function<BigfinDataRow, Signs>() { -// @Override -// public Signs apply(BigfinDataRow bigfinDataRow) { -// Signs result = bigfinDataRow.getSzClass(); -// return result; -// } -// }); -// -// // get the children of the current batch and separate them by size -// List<SpeciesBatch> vracHorsVracBatchChildren = batch.getChildBatchs(); -// Map<Serializable, SpeciesBatch> batchesBySize = new HashMap<>(); -// if (vracHorsVracBatchChildren != null) { -// batchesBySize.putAll(Maps.uniqueIndex(vracHorsVracBatchChildren, -// new Function<SpeciesBatch, Serializable>() { -// @Override -// public Serializable apply(SpeciesBatch input) { -// return input.getSampleCategoryValue(); -// } -// })); -// } -// -// // for each imported size found for the current species and vrac / hors vrac -// for (Signs size : rowsBySize.keySet()) { -// -// // get the batch with the size -// SpeciesBatch sizeBatch = batchesBySize.get(signsToCaracteristicValue.get(size)); -// // if it does not exists, create the batch -// if (sizeBatch == null) { -// sizeBatch = createSpeciesBatch(species, -// operation, -// PmfmId.SIZE_CATEGORY.getValue(), -// size, -// batch.getId()); -// } else { -// persistenceService.saveSpeciesBatchFrequency(sizeBatch.getId(), new ArrayList<SpeciesBatchFrequency>()); -// } -// -// // get the imported rows with the current species and vrac / hors vrac and size -// // and separate them by gender -// Collection<BigfinDataRow> sizeRows = rowsBySize.get(size); -// Multimap<Signs, BigfinDataRow> rowsByGender = -// Multimaps.index(sizeRows, new Function<BigfinDataRow, Signs>() { -// @Override -// public Signs apply(BigfinDataRow bigfinDataRow) { -// Signs result = bigfinDataRow.getGender(); -// return result; -// } -// }); -// test(operation, species, lengthStepPmfm, sizeBatch, PmfmId2.SEX, rowsByGender, true); -// -// -// } -// } + SampleCategoryModel sampleCategoryModel = context.getSampleCategoryModel(); + List<Integer> samplingOrder = sampleCategoryModel.getSamplingOrder(); + + List<Integer> pmfmIds = new ArrayList<>(); + pmfmIds.add(PmfmId.SORTED_UNSORTED.getValue()); + List<Function<BigfinDataRow, Signs>> functions = new ArrayList<>(); + + // put the size and order in the right order + for (Integer categoryId: samplingOrder) { + if (PmfmId.SIZE_CATEGORY.getValue().equals(categoryId)) { + pmfmIds.add(categoryId); + functions.add(new Function<BigfinDataRow, Signs>() { + @Override + public Signs apply(BigfinDataRow bigfinDataRow) { + Signs result = bigfinDataRow.getSzClass(); + return result; + } + }); + + } else if (PmfmId.SEX.getValue().equals(categoryId)) { + pmfmIds.add(categoryId); + functions.add(new Function<BigfinDataRow, Signs>() { + @Override + public Signs apply(BigfinDataRow bigfinDataRow) { + Signs result = bigfinDataRow.getGender(); + return result; + } + }); + } + } + + List<Category> categories = new ArrayList<>(); + for (int i = 0 ; i < pmfmIds.size() ; i++) { + Category category = new Category(pmfmIds.get(i), i < functions.size() ? functions.get(i) : null); + categories.add(category); + } + + Collection<SpeciesBatch> speciesBatches = batchesBySpecies.get(species); + Map<Serializable, SpeciesBatch> speciesBatchByVracHorsVrac = Maps.uniqueIndex(speciesBatches, SpeciesBatchs.GET_SAMPLE_CATEGORY_VALUE); + + BrowseBatchesParameter commonParameter = new BrowseBatchesParameter(operation, species, + lengthStepPmfm, categories, result); + browseBatchesToAddFrequencies(commonParameter, null, 0, speciesBatchByVracHorsVrac, rowsByVracHorsVrac); } addFileAsAttachment(bigfinFile, catchBatch); @@ -394,53 +322,116 @@ public class BigfinImportService extends AbstractTuttiService { return result; } - //TODO kmorin 20140905 check why data are not overriden - protected void test(FishingOperation operation, - Species species, - Caracteristic lengthStepPmfm, + /** + * Go deeper in the batches until it finds the last of gender or size class, then add the frequencies + * + * @param commonParameter The parameter containing the parameters which do not change while browsing + * @param parentBatch The parent batch (null if root) + * @param depth The depth in the batch children + * @param batchesByCaracteristic a map containing the batches by caracteristic value + * @param rowsByCaracteristic a multimap containing the rows to import by caracteristic value + */ + protected void browseBatchesToAddFrequencies(BrowseBatchesParameter commonParameter, SpeciesBatch parentBatch, - Test[] tests, int depth, + Map<Serializable, SpeciesBatch> batchesByCaracteristic, Multimap<Signs, BigfinDataRow> rowsByCaracteristic) { - // get the children of the current batch and separate them by gender - List<SpeciesBatch> batchChildren = parentBatch != null ? parentBatch.getChildBatchs() : null; - Map<Serializable, SpeciesBatch> batchesByCaracteristic = new HashMap<>(); - if (batchChildren != null) { - batchesByCaracteristic.putAll(Maps.uniqueIndex(batchChildren, - new Function<SpeciesBatch, Serializable>() { - @Override - public Serializable apply(SpeciesBatch input) { - return input.getSampleCategoryValue(); - } - })); - } - - Test test = tests[depth++]; + Category category = commonParameter.getCategories().get(depth++); for (Signs caracteristic : rowsByCaracteristic.keySet()) { Collection<BigfinDataRow> bigfinDataRows = rowsByCaracteristic.get(caracteristic); - // get the batch with the gender + // get the batch with the caracteristic SpeciesBatch batch = batchesByCaracteristic.get(signsToCaracteristicValue.get(caracteristic)); + + boolean batchHasFrequencies = false; + // if it does not exists, create the batch if (batch == null) { - batch = createSpeciesBatch(species, - operation, - test.pmfmId.getValue(), + batch = createSpeciesBatch(commonParameter.getSpecies(), + commonParameter.getOperation(), + category.getPmfmId(), caracteristic, parentBatch != null ? parentBatch.getId() : null); + + } else { + List<SpeciesBatchFrequency> frequencies = persistenceService.getAllSpeciesBatchFrequency(batch.getId()); + batchHasFrequencies = CollectionUtils.isNotEmpty(frequencies); } - if (test.function != null) { - Multimap<Signs, BigfinDataRow> rowsByNewCaracteristic = - Multimaps.index(bigfinDataRows, test.function); + // if the function is null, do not go deeper in the batch children, add the frequencies in this batch + if (category.getCategoryValueGetter() == null) { + // if the batch is not the last one, error, we cannot add the frequencies to a more categorized batch + if (CollectionUtils.isNotEmpty(batch.getChildBatchs())) { + commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.tooCategorized", + commonParameter.getSpeciesLabel(), + sizeCaracteristic.getParameterName(), + genderCaracteristic.getParameterName())); + + } else { + // create the frequencies + Integer deletedNb = persistenceService.countFrequenciesNumber( + persistenceService.getAllSpeciesBatchFrequency(batch.getId()), false); + List<SpeciesBatchFrequency> frequencies = createFrequencies(batch, bigfinDataRows, commonParameter.getLengthStepPmfm()); + persistenceService.saveSpeciesBatchFrequency(batch.getId(), frequencies); + commonParameter.getResult().incrementNbFrequenciesDeleted(deletedNb != null ? deletedNb : 0); + Integer importedNb = persistenceService.countFrequenciesNumber(frequencies, false); + commonParameter.getResult().incrementNbFrequenciesImported(importedNb != null ? importedNb : 0); + } - test(operation, species, lengthStepPmfm, batch, tests, depth, rowsByNewCaracteristic); + } else if (batchHasFrequencies) { // if the batch is supposed to be categorized again, but already has frequencies + CaracteristicQualitativeValue qualitativeValue = signsToCaracteristicValue.get(caracteristic); + commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.batch.frequenciesOnHigherLevel", + commonParameter.getSpeciesLabel(), qualitativeValue.getName())); } else { - List<SpeciesBatchFrequency> frequencies = createFrequencies(batch, bigfinDataRows, lengthStepPmfm); - persistenceService.saveSpeciesBatchFrequency(batch.getId(), frequencies); + List<SpeciesBatch> batchChildren = batch.getChildBatchs(); + + Multimap<Signs, BigfinDataRow> rowsByNewCaracteristic = Multimaps.index(bigfinDataRows, category.getCategoryValueGetter()); + + // get the children of the current batch and separate them by caracteristic + Map<Serializable, SpeciesBatch> childrenByCaracteristic = new HashMap<>(); + if (CollectionUtils.isNotEmpty(batchChildren)) { + + SpeciesBatch firstBatch = batchChildren.get(0); + Integer categoryId = firstBatch.getSampleCategoryId(); + Category nextCategory = commonParameter.getCategories().get(depth); + + // check the category is the right next one + if (!nextCategory.getPmfmId().equals(categoryId)) { + // if all the rows to import have a gender or size (the first category in the list) null equivalent + // then ok + // else error + Set<Signs> signsSet = rowsByNewCaracteristic.keySet(); + if (signsSet.size() == 1 && signsSet.iterator().next().isNullEquivalent()) {// we can go deeper + category = commonParameter.getCategories().get(depth++); + rowsByNewCaracteristic = Multimaps.index(bigfinDataRows, category.getCategoryValueGetter()); + + // check that this time, it is the right category. We can only skip one + nextCategory = commonParameter.getCategories().get(depth); + if (!nextCategory.getPmfmId().equals(categoryId)) { + commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.categoriesSkipped", + commonParameter.getSpeciesLabel(), + sizeCaracteristic.getParameterName(), + genderCaracteristic.getParameterName() + )); + continue; + } + + } else { + commonParameter.getResult().addError(t("tutti.service.bigfinImport.error.species.categorySkipped", + commonParameter.getSpeciesLabel(), + persistenceService.getCaracteristic(nextCategory.getPmfmId()).getParameterName() + )); + continue; + } + } + childrenByCaracteristic.putAll(Maps.uniqueIndex(batchChildren, SpeciesBatchs.GET_SAMPLE_CATEGORY_VALUE)); + } + + // go deeper in the batch children + browseBatchesToAddFrequencies(commonParameter, batch, depth, childrenByCaracteristic, rowsByNewCaracteristic); } } @@ -531,13 +522,83 @@ public class BigfinImportService extends AbstractTuttiService { persistenceService.createAttachment(attachment, f); } - protected class Test { - PmfmId pmfmId; - Function<BigfinDataRow, Signs> function; + /** + * Class describing the category of a batch level and the function to separate the rows to import by category value + */ + private class Category { - public Test(PmfmId pmfmId, Function<BigfinDataRow, Signs> function) { + private Integer pmfmId; + /** function to get the value of the caracteristic we want to group the batches by (eg size or gender) */ + private Function<BigfinDataRow, Signs> categoryValueGetter; + + public Category(Integer pmfmId, Function<BigfinDataRow, Signs> dataGetter) { this.pmfmId = pmfmId; - this.function = function; + this.categoryValueGetter = dataGetter; + } + + public Integer getPmfmId() { + return pmfmId; + } + + public Function<BigfinDataRow, Signs> getCategoryValueGetter() { + return categoryValueGetter; + } + } + + /** + * Class containing the common parameters for the batch browsing + * These parameter do not change when you go deeper. + */ + private class BrowseBatchesParameter { + /** the current fishing operation */ + private FishingOperation operation; + /** the current species */ + private Species species; + /** the lengthstep caracteristic found in the protocol */ + private Caracteristic lengthStepPmfm; + /** the ordered categories */ + private List<Category> categories; + /** the result of the import (to add the errors) */ + private BigfinImportResult result; + /** label for the species in the errors */ + private String speciesLabel; + + public BrowseBatchesParameter(FishingOperation operation, Species species, Caracteristic lengthStepPmfm, + List<Category> categories, BigfinImportResult result) { + this.operation = operation; + this.species = species; + this.lengthStepPmfm = lengthStepPmfm; + this.categories = categories; + this.result = result; + + speciesLabel = species.getSurveyCode(); + if (StringUtils.isBlank(speciesLabel)) { + speciesLabel = species.getRefTaxCode(); + } + } + + public FishingOperation getOperation() { + return operation; + } + + public Species getSpecies() { + return species; + } + + public Caracteristic getLengthStepPmfm() { + return lengthStepPmfm; + } + + public List<Category> getCategories() { + return categories; + } + + public BigfinImportResult getResult() { + return result; + } + + public String getSpeciesLabel() { + return speciesLabel; } } } diff --git a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java index 169a14d..208325e 100644 --- a/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java +++ b/tutti-service/src/main/java/fr/ifremer/tutti/service/bigfin/Signs.java @@ -50,6 +50,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.NON_SEXED_SEX.getValue(); } + + @Override + public boolean isNullEquivalent() { + return true; + } }, MALE("M") { @Override @@ -61,6 +66,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.SEX_MALE.getValue(); } + + @Override + public boolean isNullEquivalent() { + return false; + } }, FEMALE("F") { @Override @@ -72,6 +82,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.SEX_FEMALE.getValue(); } + + @Override + public boolean isNullEquivalent() { + return false; + } }, // classe de taille, 1 = petit ; 2 = gros ; 0 = pas de classe de taille (saisie libre donc risque fort de mauvaise saisie) @@ -85,6 +100,12 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId2.NOT_SIZED.getValue(); } + + @Override + public boolean isNullEquivalent() { + return true; + } + }, SMALL("1") { @Override @@ -96,6 +117,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.SIZE_SMALL.getValue(); } + + @Override + public boolean isNullEquivalent() { + return false; + } }, BIG("2") { @Override @@ -107,6 +133,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.SIZE_BIG.getValue(); } + + @Override + public boolean isNullEquivalent() { + return false; + } }, // vrac / hors vrac @@ -120,6 +151,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.SORTED_VRAC.getValue(); } + + @Override + public boolean isNullEquivalent() { + return false; + } }, HORS_VRAC("HV") { @Override @@ -131,6 +167,11 @@ public enum Signs { public Integer getQualitativeValueId() { return QualitativeValueId.SORTED_HORS_VRAC.getValue(); } + + @Override + public boolean isNullEquivalent() { + return false; + } }; private String sign; @@ -163,6 +204,9 @@ public enum Signs { public abstract Integer getQualitativeValueId(); + // if true, can use this value in the import file to replace a skipped category + public abstract boolean isNullEquivalent(); + public void registerSign(Caracteristic caracteristic, Map<Signs, CaracteristicQualitativeValue> map) { Integer valueId = getQualitativeValueId(); diff --git a/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties b/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties index 3dcb9bf..0923c2d 100644 --- a/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties +++ b/tutti-service/src/main/resources/i18n/tutti-service_en_GB.properties @@ -47,8 +47,13 @@ tutti.report.step.generateReport= tutti.report.step.load.fishingOperation= tutti.service.arp.import.attachment.comment= tutti.service.bigfin.import.attachment.comment= +tutti.service.bigfinImport.error.species.batch.frequenciesOnHigherLevel= +tutti.service.bigfinImport.error.species.categoriesSkipped= +tutti.service.bigfinImport.error.species.categorySkipped= tutti.service.bigfinImport.error.species.not.found= +tutti.service.bigfinImport.error.species.tooCategorized= tutti.service.bigfinImport.error.species.without.lengthstep= +tutti.service.bigfinImport.error.species.wrongNextCategory= tutti.service.bigfinImport.error.szClass.unknwon= tutti.service.bigfinimport.error.no.protocol= tutti.service.compressZipFile.error= diff --git a/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties b/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties index 5b7b04b..bd22b91 100644 --- a/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties +++ b/tutti-service/src/main/resources/i18n/tutti-service_fr_FR.properties @@ -44,7 +44,11 @@ tutti.report.step.export.fishingOperation=Exporter le trait sélectionné tutti.report.step.generateReport=Générer le rapport tutti.report.step.load.fishingOperation=Charger le trait sélectionné tutti.service.bigfin.import.attachment.comment=Import Bigfin du %s +tutti.service.bigfinImport.error.species.batch.frequenciesOnHigherLevel=Le lot de '<strong>%1s / %2s</strong>' a déjà des mensurations +tutti.service.bigfinImport.error.species.categoriesSkipped=L'espèce '<strong>%1s</strong>' a été catégorisée sans les catégories '<strong>%2s</strong>' et '<strong>%3s</strong>' +tutti.service.bigfinImport.error.species.categorySkipped=L'espèce '<strong>%1s</strong>' a été catégorisée sans la catégorie '<strong>%2s</strong>' tutti.service.bigfinImport.error.species.not.found=L'espèce '<strong>%s</strong>' est inconnue +tutti.service.bigfinImport.error.species.tooCategorized=L'espèce '<strong>%1s</strong>' est trop catégorisée (pas limitée à '<strong>%2s</strong>' et '<strong>%3s</strong>') tutti.service.bigfinImport.error.species.without.lengthstep=L'espèce '<strong>%s</strong>' n'a pas de classe de taille associée dans le protocole. tutti.service.bigfinImport.error.szClass.unknwon=Ligne <i>%s</i>, code inconnu (doit être 0, 1 ou 2) tutti.service.bigfinimport.error.no.protocol=Impossible de faire un import Bigfin sans protocol. diff --git a/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java b/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java index 5846a42..035d440 100644 --- a/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java +++ b/tutti-service/src/test/java/fr/ifremer/tutti/service/bigfin/BigfinImportServiceTest.java @@ -99,40 +99,39 @@ public class BigfinImportServiceTest { BigfinImportResult importResult = service.importFile(importFile, operation, catchBatch); - int nbSortedAdded = importResult.getNbVracImported(); - int nbUnsortedAdded = importResult.getNbHorsVracImported(); + int nbFrequenciesAdded = importResult.getNbFrequenciesImported(); + List<String> fatals = importResult.getFatalErrors(); List<String> errors = importResult.getErrors(); if (log.isInfoEnabled()) { - log.info("Sorted Imported: " + nbSortedAdded); - log.info("Unsorted Imported: " + nbUnsortedAdded); + log.info("Frequencies Imported: " + nbFrequenciesAdded); + log.info("Fatals: " + fatals.size()); log.info("Errors: " + errors.size()); } - int nbNewSortedBatchs = 3; - int nbNewUnsortedBatchs = 0; - Assert.assertEquals(nbNewSortedBatchs, nbSortedAdded); - Assert.assertEquals(nbNewUnsortedBatchs, nbUnsortedAdded); + int nbNewFrequencies = 3; + Assert.assertEquals(nbNewFrequencies, nbFrequenciesAdded); + Assert.assertEquals(0, fatals.size()); Assert.assertEquals(0, errors.size()); // no batch imported BatchContainer<SpeciesBatch> rootSpeciesBatchAfter = persistenceService.getRootSpeciesBatch(operation.getId(), false); - int totalSortedBatchs = 0; - int totalUnsortedBatchs = 0; - for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { - - boolean sorted = vracPredicate.apply(speciesBatch); - - if (sorted) { - totalSortedBatchs++; - } else { - totalUnsortedBatchs++; - } - } - - Assert.assertEquals(nbNewSortedBatchs, totalSortedBatchs); - Assert.assertEquals(nbNewUnsortedBatchs, totalUnsortedBatchs); +// int totalSortedBatchs = 0; +// int totalUnsortedBatchs = 0; +// for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { +// +// boolean sorted = vracPredicate.apply(speciesBatch); +// +// if (sorted) { +// totalSortedBatchs++; +// } else { +// totalUnsortedBatchs++; +// } +// } +// +// Assert.assertEquals(nbNewSortedBatchs, totalSortedBatchs); +// Assert.assertEquals(nbNewUnsortedBatchs, totalUnsortedBatchs); } @Test @@ -151,40 +150,38 @@ public class BigfinImportServiceTest { BigfinImportResult importResult = service.importFile(importFile, operation, catchBatch); - int nbSortedAdded = importResult.getNbVracImported(); - int nbUnsortedAdded = importResult.getNbHorsVracImported(); + int nbFrequenciesAdded = importResult.getNbFrequenciesImported(); + List<String> fatals = importResult.getFatalErrors(); List<String> errors = importResult.getErrors(); if (log.isInfoEnabled()) { - log.info("Sorted Imported: " + nbSortedAdded); - log.info("Unsorted Imported: " + nbUnsortedAdded); + log.info("Frequencies Imported: " + nbFrequenciesAdded); + log.info("Fatals: " + fatals.size()); log.info("Errors: " + errors.size()); } - int nbNewSortedBatchs = 0; - int nbNewUnsortedBatchs = 0; - int nbErrors = 3; - Assert.assertEquals(nbNewSortedBatchs, nbSortedAdded); - Assert.assertEquals(nbNewUnsortedBatchs, nbUnsortedAdded); + int nbNewFrequencies = 0; + int nbFatals = 3; + int nbErrors = 0; + Assert.assertEquals(nbNewFrequencies, nbFrequenciesAdded); + Assert.assertEquals(nbFatals, fatals.size()); Assert.assertEquals(nbErrors, errors.size()); // no batch imported BatchContainer<SpeciesBatch> rootSpeciesBatchAfter = persistenceService.getRootSpeciesBatch(operation.getId(), false); - int totalSortedBatchs = 0; - int totalUnsortedBatchs = 0; - for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { - - boolean sorted = vracPredicate.apply(speciesBatch); - - if (sorted) { - totalSortedBatchs++; - } else { - totalUnsortedBatchs++; - } - } - - Assert.assertEquals(nbNewSortedBatchs, totalSortedBatchs); - Assert.assertEquals(nbNewUnsortedBatchs, totalUnsortedBatchs); +// int totalSortedBatchs = 0; +// for (SpeciesBatch speciesBatch : rootSpeciesBatchAfter.getChildren()) { +// +// boolean sorted = vracPredicate.apply(speciesBatch); +// +// if (sorted) { +// totalSortedBatchs++; +// } else { +// totalUnsortedBatchs++; +// } +// } + +// Assert.assertEquals(nbFrequenciesAdded, totalSortedBatchs); } } diff --git a/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java b/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java index e615f14..cf4aa9b 100644 --- a/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java +++ b/tutti-ui-swing/src/main/java/fr/ifremer/tutti/ui/swing/action/ImportBigfinAction.java @@ -116,20 +116,35 @@ public class ImportBigfinAction extends AbstractTuttiAction<SpeciesBatchUIModel, if (importResult.isDone()) { sendMessage(t("tutti.editSpeciesBatch.action.importBigfin.success", - importResult.getNbVracImported(), importResult.getNbHorsVracImported())); + importResult.getNbFrequenciesImported(), + importResult.getNbFrequenciesDeleted())); + + if (!importResult.getErrors().isEmpty()) { + StringBuilder sb = new StringBuilder(); + for (String s : importResult.getErrors()) { + sb.append("<li>").append(s).append("</li>"); + } + displayWarningMessage( + t("tutti.editSpeciesBatch.action.importBigfin.errors.fishingOperation.title"), + "<html><body>" + + t("tutti.editSpeciesBatch.action.importBigfin.errors.fishingOperation", sb.toString()) + + "</body></html>" + ); + } + } else { StringBuilder sb = new StringBuilder(); - for (String s : importResult.getErrors()) { + for (String s : importResult.getFatalErrors()) { sb.append("<li>").append(s).append("</li>"); } displayWarningMessage( - t("tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation.title"), + t("tutti.editSpeciesBatch.action.importBigfin.fatal.fishingOperation.title"), "<html><body>" + - t("tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation", sb.toString()) + + t("tutti.editSpeciesBatch.action.importBigfin.fatal.fishingOperation", sb.toString()) + "</body></html>" ); - sendMessage(t("tutti.editSpeciesBatch.action.importBigfin.no.matching.data")); + sendMessage(t("tutti.editSpeciesBatch.action.importBigfin.fatal.data")); } } } diff --git a/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_en_GB.properties b/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_en_GB.properties index f25bc22..42a087a 100644 --- a/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_en_GB.properties +++ b/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_en_GB.properties @@ -993,6 +993,11 @@ tutti.editSpeciesBatch.action.ichtyometer= tutti.editSpeciesBatch.action.ichtyometer.mnemonic= tutti.editSpeciesBatch.action.ichtyometer.tip= tutti.editSpeciesBatch.action.importBigfin= +tutti.editSpeciesBatch.action.importBigfin.errors.fishingOperation= +tutti.editSpeciesBatch.action.importBigfin.errors.fishingOperation.title= +tutti.editSpeciesBatch.action.importBigfin.fatal.data= +tutti.editSpeciesBatch.action.importBigfin.fatal.fishingOperation= +tutti.editSpeciesBatch.action.importBigfin.fatal.fishingOperation.title= tutti.editSpeciesBatch.action.importBigfin.mnemonic= tutti.editSpeciesBatch.action.importBigfin.no.matching.data= tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation= diff --git a/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties b/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties index ec3417b..9259209 100644 --- a/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties +++ b/tutti-ui-swing/src/main/resources/i18n/tutti-ui-swing_fr_FR.properties @@ -977,11 +977,13 @@ tutti.editSpeciesBatch.action.exportMultiPost.mnemonic=E tutti.editSpeciesBatch.action.exportMultiPost.success=Les lots d'espèces ont été exportés dans le fichier %s tutti.editSpeciesBatch.action.exportMultiPost.tip=Exporter les lots d'espèces pour les importer sur le poste maître tutti.editSpeciesBatch.action.importBigfin=Import Bigfin +tutti.editSpeciesBatch.action.importBigfin.errors.fishingOperation=L'import Bigfin a été réalisé, mais des erreurs ont été détectées \:<ul>%s</ul><br/> +tutti.editSpeciesBatch.action.importBigfin.errors.fishingOperation.title=Import Bigfin +tutti.editSpeciesBatch.action.importBigfin.fatal.data=Import Bigfin non réalisé (des erreurs ont été détectées lors de la lecture du fichier) +tutti.editSpeciesBatch.action.importBigfin.fatal.fishingOperation=L'import Bigfin n'a pas été réalisé, des erreurs ont été détectées \:<ul>%s</ul><br/>Aucun lot n'a donc été importé. +tutti.editSpeciesBatch.action.importBigfin.fatal.fishingOperation.title=Import Bigfin tutti.editSpeciesBatch.action.importBigfin.mnemonic=B -tutti.editSpeciesBatch.action.importBigfin.no.matching.data=Import Bigfin non réalisé (des erreurs ont été détectées lors de la lecture du fichier) -tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation=L'import Bigfin n'a pas été réalisé, des erreurs ont été détectées \:<ul>%s</ul><br/>Aucun lot n'a donc été importé. -tutti.editSpeciesBatch.action.importBigfin.no.matching.fishingOperation.title=Import Bigfin -tutti.editSpeciesBatch.action.importBigfin.success=Import Bigfin réussi \: %1s espèces importées (Vrac), %2s espèces importées (Hors-Vrac) +tutti.editSpeciesBatch.action.importBigfin.success=Import Bigfin réussi \: %1s mensurations importées, %2s mensurations supprimées tutti.editSpeciesBatch.action.importBigfin.tip=Import Bigfin tutti.editSpeciesBatch.action.importMultiPost=Importer des lots d'espèces tutti.editSpeciesBatch.action.importMultiPost.mnemonic=I -- To stop receiving notification emails like this one, please contact codelutin.com SCM administrator <admin+scm@codelutin.com>.
participants (1)
-
codelutin.com scm