diff --git a/li.strolch.rest/src/main/java/li/strolch/rest/endpoint/ReportResource.java b/li.strolch.rest/src/main/java/li/strolch/rest/endpoint/ReportResource.java index 3c1358cb8..118d8f8cd 100644 --- a/li.strolch.rest/src/main/java/li/strolch/rest/endpoint/ReportResource.java +++ b/li.strolch.rest/src/main/java/li/strolch/rest/endpoint/ReportResource.java @@ -2,8 +2,8 @@ package li.strolch.rest.endpoint; import static java.util.Comparator.comparing; import static li.strolch.model.StrolchModelConstants.BAG_PARAMETERS; -import static li.strolch.model.StrolchModelConstants.TYPE_PARAMETERS; import static li.strolch.report.ReportConstants.*; +import static li.strolch.rest.RestfulStrolchComponent.getInstance; import static li.strolch.rest.StrolchRestfulConstants.PARAM_DATE_RANGE_SEL; import static li.strolch.rest.StrolchRestfulConstants.*; import static li.strolch.utils.helper.StringHelper.*; @@ -42,7 +42,6 @@ import li.strolch.privilege.model.SimpleRestrictable; import li.strolch.report.Report; import li.strolch.report.ReportElement; import li.strolch.report.ReportSearch; -import li.strolch.rest.RestfulStrolchComponent; import li.strolch.rest.StrolchRestfulConstants; import li.strolch.rest.helper.ResponseUtil; import li.strolch.utils.ObjectHelper; @@ -72,9 +71,9 @@ public class ReportResource { Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE); if (isEmpty(realm)) - realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm(); + realm = getInstance().getContainer().getRealm(cert).getRealm(); - try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext())) { + try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext())) { StrolchRootElementToJsonVisitor visitor = new StrolchRootElementToJsonVisitor().flat().withoutVersion() .withoutObjectType().withoutPolicies().withoutStateVariables() @@ -97,7 +96,7 @@ public class ReportResource { Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE); if (isEmpty(realm)) - realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm(); + realm = getInstance().getContainer().getRealm(cert).getRealm(); int limit = isNotEmpty(limitS) ? Integer.parseInt(limitS) : 10; @@ -110,8 +109,10 @@ public class ReportResource { localeJ = localesJ.get(cert.getLocale().toLanguageTag()).getAsJsonObject(); } - JsonArray result = new JsonArray(); - try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext()); + long start = System.nanoTime(); + + JsonObject result = new JsonObject(); + try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext()); Report report = new Report(tx, id)) { tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), id)); @@ -120,10 +121,12 @@ public class ReportResource { if (localeJ != null) report.getReportPolicy().setI18nData(localeJ); - MapOfSets criteria = report.generateFilterCriteria(limit); - List types = new ArrayList<>(criteria.keySet()); + JsonArray facetsJ = new JsonArray(); JsonObject finalLocaleJ = localeJ; - types.stream().sorted(comparing(type -> { + + MapOfSets criteria = report.generateFilterCriteria(limit); + + criteria.keySet().stream().sorted(comparing(type -> { JsonElement translatedJ = finalLocaleJ == null ? null : finalLocaleJ.get(type); return translatedJ == null ? type : translatedJ.getAsString(); })).forEach(type -> { @@ -136,9 +139,16 @@ public class ReportResource { o.addProperty(Tags.Json.NAME, f.getName()); return o; }).collect(JsonArray::new, JsonArray::add, JsonArray::addAll)); - result.add(filter); + facetsJ.add(filter); }); + String duration = formatNanoDuration(System.nanoTime() - start); + + result.add(PARAM_FACETS, facetsJ); + result.addProperty(PARAM_DURATION, duration); + result.addProperty(PARAM_PARALLEL, report.isParallel()); + + logger.info("Facet Generation for " + id + " took: " + duration); return ResponseUtil.toResponse(DATA, result); } } @@ -153,7 +163,7 @@ public class ReportResource { Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE); if (isEmpty(realm)) - realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm(); + realm = getInstance().getContainer().getRealm(cert).getRealm(); String query = isNotEmpty(queryS) ? queryS.toLowerCase() : queryS; int limit = isNotEmpty(limitS) ? Integer.parseInt(limitS) : 10; @@ -167,7 +177,9 @@ public class ReportResource { localeJ = localesJ.get(cert.getLocale().toLanguageTag()).getAsJsonObject(); } - try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext()); + long start = System.nanoTime(); + + try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext()); Report report = new Report(tx, id)) { tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), id)); @@ -183,14 +195,31 @@ public class ReportResource { criteria = criteria.filter(f -> ObjectHelper.contains(f.getName(), parts, true)); } + int maxFacetValues; + int reportMaxFacetValues = report.getReportResource().getInteger(PARAM_MAX_FACET_VALUES); + if (reportMaxFacetValues != 0 && reportMaxFacetValues != limit) { + logger.warn("Report " + report.getReportResource().getId() + " has " + PARAM_MAX_FACET_VALUES + + " defined as " + reportMaxFacetValues + ". Ignoring requested limit " + limit); + maxFacetValues = reportMaxFacetValues; + } else { + maxFacetValues = limit; + } + + criteria = criteria.sorted(comparing(StrolchElement::getName)); + + if (maxFacetValues != 0) + criteria = criteria.limit(maxFacetValues); + // add the data finally - JsonArray array = criteria.limit(limit).sorted(comparing(StrolchElement::getName)).map(f -> { + JsonArray array = criteria.map(f -> { JsonObject o = new JsonObject(); o.addProperty(Tags.Json.ID, f.getId()); o.addProperty(Tags.Json.NAME, f.getName()); return o; }).collect(JsonArray::new, JsonArray::add, JsonArray::addAll); + String duration = formatNanoDuration(System.nanoTime() - start); + logger.info("Facet Generation for " + id + "." + type + " took: " + duration); return ResponseUtil.toResponse(DATA, array); } } @@ -204,7 +233,7 @@ public class ReportResource { Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE); if (isEmpty(realm)) - realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm(); + realm = getInstance().getContainer().getRealm(cert).getRealm(); DBC.PRE.assertNotEmpty("report ID is required", id); @@ -255,7 +284,7 @@ public class ReportResource { long start = System.nanoTime(); - try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext()); + try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext()); Report report = new Report(tx, id)) { tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), id)); @@ -337,7 +366,7 @@ public class ReportResource { Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE); if (isEmpty(realm)) - realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm(); + realm = getInstance().getContainer().getRealm(cert).getRealm(); DBC.PRE.assertNotEmpty("report ID is required", id); @@ -398,7 +427,7 @@ public class ReportResource { return out -> { - try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext()); + try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext()); Report report = new Report(tx, reportId)) { tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), reportId)); diff --git a/li.strolch.service/src/main/java/li/strolch/report/Report.java b/li.strolch.service/src/main/java/li/strolch/report/Report.java index 605f7f099..4d831b492 100644 --- a/li.strolch.service/src/main/java/li/strolch/report/Report.java +++ b/li.strolch.service/src/main/java/li/strolch/report/Report.java @@ -35,6 +35,10 @@ public class Report implements AutoCloseable { return this.reportPolicy; } + public Resource getReportResource() { + return this.reportPolicy.getReportResource(); + } + public boolean isParallel() { return this.reportPolicy.isParallel(); } diff --git a/li.strolch.service/src/main/java/li/strolch/report/ReportConstants.java b/li.strolch.service/src/main/java/li/strolch/report/ReportConstants.java index 910691c7a..4bdf5330b 100644 --- a/li.strolch.service/src/main/java/li/strolch/report/ReportConstants.java +++ b/li.strolch.service/src/main/java/li/strolch/report/ReportConstants.java @@ -15,12 +15,14 @@ public class ReportConstants { public static final String PARAM_PARALLEL = "parallel"; public static final String PARAM_DESCENDING = "descending"; public static final String PARAM_DATE_RANGE_SEL = "dateRangeSel"; + public static final String PARAM_DIRECT_CRITERIA = "directCriteria"; public static final String PARAM_FIELD_REF = "fieldRef"; public static final String PARAM_FIELD_REF1 = "fieldRef1"; public static final String PARAM_FIELD_REF2 = "fieldRef2"; public static final String PARAM_ALLOW_MISSING_COLUMNS = "allowMissingColumns"; public static final String PARAM_FILTER_MISSING_VALUES_AS_TRUE = "filterMissingValuesAsTrue"; public static final String PARAM_MAX_ROWS_FOR_FACET_GENERATION = "maxRowsForFacetGeneration"; + public static final String PARAM_MAX_FACET_VALUES = "maxFacetValues"; public static final String PARAM_POLICY = "policy"; public static final String PARAM_JOIN_PARAM = "joinParam"; diff --git a/li.strolch.service/src/main/java/li/strolch/report/policy/GenericReport.java b/li.strolch.service/src/main/java/li/strolch/report/policy/GenericReport.java index da5d54d77..2177d37a9 100644 --- a/li.strolch.service/src/main/java/li/strolch/report/policy/GenericReport.java +++ b/li.strolch.service/src/main/java/li/strolch/report/policy/GenericReport.java @@ -40,11 +40,13 @@ import li.strolch.utils.iso8601.ISO8601; */ public class GenericReport extends ReportPolicy { + public static final int MAX_FACET_VALUE_LIMIT = 100; protected Resource reportRes; protected ParameterBag columnsBag; protected List orderingParams; protected Map filterCriteriaParams; + protected Set directCriteria; protected boolean parallel; protected boolean descending; protected boolean allowMissingColumns; @@ -185,6 +187,14 @@ public class GenericReport extends ReportPolicy { this.filtersByPolicy.put(filterFunction, refTuple); } + + // get the list of types of criteria to query directly, not over the element stream + this.directCriteria = new HashSet<>(this.reportRes.getStringList(PARAM_DIRECT_CRITERIA)); + } + + @Override + public Resource getReportResource() { + return this.reportRes; } public boolean isDescending() { @@ -336,6 +346,17 @@ public class GenericReport extends ReportPolicy { */ @Override public Stream> buildStream() { + return buildStream(true); + } + + /** + * Builds the stream of rows on which further transformations can be performed. Each row is a {@link Map} for where + * the key is an element type, and the value is the associated element + * + * @return this for chaining + */ + @Override + public Stream> buildStream(boolean withOrdering) { Stream> stream; @@ -354,7 +375,7 @@ public class GenericReport extends ReportPolicy { stream = stream.peek(e -> incrementCounter()); - if (hasOrdering()) + if (withOrdering && hasOrdering()) stream = stream.sorted(this::sort); return stream; @@ -442,7 +463,7 @@ public class GenericReport extends ReportPolicy { "Additional join type " + joinWithP.getUom() + " is not available on row for " + joinWithP.getLocator()); - Optional> refP = lookupParameter(joinWithP, joinElement); + Optional> refP = lookupParameter(joinWithP, joinElement, false); if (refP.isEmpty()) { throw new IllegalStateException( "Parameter reference (" + joinWithP.getValue() + ") for " + joinWithP.getLocator() @@ -517,17 +538,32 @@ public class GenericReport extends ReportPolicy { * Generates the filter criteria for this report, i.e. it returns a {@link MapOfSets} which defines the type of * elements on which a filter can be set and the {@link Set} of IDs which can be used for filtering. * + * @param limit + * the max number of values per filter criteria to return + * * @return the filter criteria as a map of sets */ @Override public MapOfSets generateFilterCriteria(int limit) { + if (limit <= 0 || limit >= MAX_FACET_VALUE_LIMIT) { + logger.warn("Overriding invalid limit " + limit + " with " + MAX_FACET_VALUE_LIMIT); + limit = 100; + } + + int maxFacetValues; + int reportMaxFacetValues = this.reportRes.getInteger(PARAM_MAX_FACET_VALUES); + if (reportMaxFacetValues != 0 && reportMaxFacetValues != limit) { + logger.warn("Report " + this.reportRes.getId() + " has " + PARAM_MAX_FACET_VALUES + " defined as " + + reportMaxFacetValues + ". Ignoring requested limit " + limit); + maxFacetValues = reportMaxFacetValues; + } else { + maxFacetValues = limit; + } + MapOfSets result = new MapOfSets<>(true); - Iterator> iter = buildStream().iterator(); - if (!iter.hasNext()) - return result; - + // we need the list of possible element types, which designate the criteria List criteria = this.filterCriteriaParams.values().stream() // .filter(p -> { if (p.getUom().equals(UOM_NONE)) @@ -538,34 +574,72 @@ public class GenericReport extends ReportPolicy { return filterCriteriaAllowed(p.getId()); }) // .sorted(comparing(StringParameter::getIndex)) // - .map(StringParameter::getUom).collect(toList()); + .map(StringParameter::getUom) // + .collect(toList()); + + int maxRowsForFacetGeneration = this.reportRes.getInteger(PARAM_MAX_ROWS_FOR_FACET_GENERATION); + + if (!this.directCriteria.isEmpty()) { + criteria.forEach(type -> { + if (!this.directCriteria.contains(type)) + return; + StringParameter filterCriteriaP = this.filterCriteriaParams.get(type); + Stream stream; + switch (filterCriteriaP.getInterpretation()) { + case INTERPRETATION_RESOURCE_REF: + stream = tx().streamResources(filterCriteriaP.getUom()); + break; + case INTERPRETATION_ORDER_REF: + stream = tx().streamOrders(filterCriteriaP.getUom()); + break; + case INTERPRETATION_ACTIVITY_REF: + stream = tx().streamActivities(filterCriteriaP.getUom()); + break; + default: + throw new IllegalArgumentException("Unhandled element type " + filterCriteriaP.getInterpretation()); + } + + stream = stream.map(this::mapFilterCriteria).filter(this::filterDirectCriteria); + + if (hasOrdering()) + stream = stream.sorted(this::sortDirectCriteria); + + if (maxFacetValues > 0) + stream = stream.limit(maxFacetValues); + + stream.forEachOrdered(e -> result.addElement(e.getType(), e)); + }); + + criteria.removeAll(this.directCriteria); + } + + Stream> stream = buildStream(false); + if (maxRowsForFacetGeneration > 0) + stream = stream.limit(maxRowsForFacetGeneration); + + Iterator> iter = stream.iterator(); - boolean maxRowsForFacetGeneration = this.reportRes.getBoolean(PARAM_MAX_ROWS_FOR_FACET_GENERATION); - long count = 0; while (iter.hasNext()) { Map row = iter.next(); for (String criterion : criteria) { - if (row.containsKey(criterion)) { - if (result.size(criterion) >= limit) - continue; - + if (row.containsKey(criterion) && result.size(criterion) < maxFacetValues) result.addElement(criterion, row.get(criterion)); - } } // stop if we have enough data - count++; -// if (trimFacetValues) { -// -// } - if (result.stream().mapToInt(e -> e.getValue().size()).allMatch(v -> v >= limit)) + if (result.stream().filter(e -> !this.directCriteria.contains(e.getKey())) + .mapToInt(e -> e.getValue().size()).allMatch(v -> v >= maxFacetValues)) break; } return result; } + protected StrolchRootElement mapFilterCriteria(StrolchRootElement element) { + return element; + } + @Override public Stream generateFilterCriteria(String type) { return buildStream().filter(row -> row.containsKey(type)).map(row -> row.get(type)).distinct(); @@ -580,6 +654,56 @@ public class GenericReport extends ReportPolicy { return this.orderingParams != null && !this.orderingParams.isEmpty(); } + protected int sortDirectCriteria(StrolchRootElement column1, StrolchRootElement column2) { + if (column1 == null && column2 == null) + return 0; + if (column1 == null) + return -1; + if (column2 == null) + return 1; + + for (StringParameter fieldRefP : this.orderingParams) { + String type = fieldRefP.getUom(); + + if (!column1.getType().equals(type)) + continue; + + int sortVal; + if (fieldRefP.getValue().startsWith("$")) { + Object columnValue1 = evaluateColumnValue(fieldRefP, Map.of(column1.getType(), column1), false); + Object columnValue2 = evaluateColumnValue(fieldRefP, Map.of(column2.getType(), column2), false); + + if (this.descending) { + sortVal = ObjectHelper.compare(columnValue2, columnValue1, true); + } else { + sortVal = ObjectHelper.compare(columnValue1, columnValue2, true); + } + + } else { + Optional> param1 = lookupParameter(fieldRefP, column1, false); + Optional> param2 = lookupParameter(fieldRefP, column2, false); + + if (param1.isEmpty() && param2.isEmpty()) + continue; + + if (param1.isPresent() && param2.isEmpty()) + return 1; + else if (param1.isEmpty()) + return -1; + + if (this.descending) + sortVal = param2.get().compareTo(param1.get()); + else + sortVal = param1.get().compareTo(param2.get()); + } + + if (sortVal != 0) + return sortVal; + } + + return 0; + } + /** * Implements a sorting of the given two rows. This implementation using the ordering as is defined in {@link * ReportConstants#BAG_ORDERING} @@ -618,8 +742,8 @@ public class GenericReport extends ReportPolicy { } } else { - Optional> param1 = lookupParameter(fieldRefP, column1); - Optional> param2 = lookupParameter(fieldRefP, column2); + Optional> param1 = lookupParameter(fieldRefP, column1, false); + Optional> param2 = lookupParameter(fieldRefP, column2, false); if (param1.isEmpty() && param2.isEmpty()) continue; @@ -653,6 +777,39 @@ public class GenericReport extends ReportPolicy { && !this.filtersById.isEmpty()); } + protected boolean filterDirectCriteria(StrolchRootElement element) { + + // do filtering by policies + for (ReportFilterPolicy filterPolicy : this.filtersByPolicy.keySet()) { + TypedTuple refTuple = this.filtersByPolicy.get(filterPolicy); + if (refTuple.hasBoth()) { + // not applicable for direct criteria + continue; + } + + // not for this element + if (!refTuple.getFirst().getUom().equals(element.getType())) + continue; + + Object value = evaluateColumnValue(refTuple.getFirst(), Map.of(element.getType(), element), true); + if (this.filterMissingValuesAsTrue && value == null) + continue; + if (value == null || !filterPolicy.filter(value)) + return false; + } + + // then we do a filter by criteria + if (this.filtersById != null && !this.filtersById.isEmpty() && this.filtersById.containsSet( + element.getType())) { + + if (!this.filtersById.getSet(element.getType()).contains(element.getId())) + return false; + } + + // otherwise we want to keep this row + return true; + } + /** * Returns true if the element is filtered, i.e. is to be kep, false if it should not be kept in the stream * @@ -667,7 +824,7 @@ public class GenericReport extends ReportPolicy { for (ReportFilterPolicy filterPolicy : this.filtersByPolicy.keySet()) { TypedTuple refTuple = this.filtersByPolicy.get(filterPolicy); - if (refTuple.hasFirst() && refTuple.hasSecond()) { + if (refTuple.hasBoth()) { Object value1 = evaluateColumnValue(refTuple.getFirst(), row, true); Object value2 = evaluateColumnValue(refTuple.getSecond(), row, true); @@ -702,7 +859,7 @@ public class GenericReport extends ReportPolicy { if (dateRangeSel.equals(COL_DATE)) { date = element.accept(new ElementZdtDateVisitor()); } else { - Optional> param = lookupParameter(this.dateRangeSelP, element); + Optional> param = lookupParameter(this.dateRangeSelP, element, false); if (param.isEmpty() || param.get().getValueType() != StrolchValueType.DATE) throw new IllegalStateException( "Date Range selector is invalid, as referenced parameter is not a Date but " @@ -776,7 +933,8 @@ public class GenericReport extends ReportPolicy { else columnValue = parameter; } else { - columnValue = lookupParameter(columnDefP, column).orElseGet(() -> allowNull ? null : new StringParameter()); + columnValue = lookupParameter(columnDefP, column, allowNull) // + .orElseGet(() -> allowNull ? null : new StringParameter()); } return columnValue; @@ -828,7 +986,8 @@ public class GenericReport extends ReportPolicy { * * @return the {@link Optional} with the parameter */ - protected Optional> lookupParameter(StringParameter paramRefP, StrolchRootElement element) { + protected Optional> lookupParameter(StringParameter paramRefP, StrolchRootElement element, + boolean overrideAllowMissingColumns) { String paramRef = paramRefP.getValue(); String[] locatorParts = paramRef.split(Locator.PATH_SEPARATOR); @@ -841,7 +1000,7 @@ public class GenericReport extends ReportPolicy { String paramKey = locatorParts[2]; Parameter param = element.getParameter(bagKey, paramKey); - if (!allowMissingColumns && param == null) + if (!overrideAllowMissingColumns && !this.allowMissingColumns && param == null) throw new IllegalStateException( "Parameter reference (" + paramRef + ") for " + paramRefP.getLocator() + " not found on " + element.getLocator()); diff --git a/li.strolch.service/src/main/java/li/strolch/report/policy/ReportPolicy.java b/li.strolch.service/src/main/java/li/strolch/report/policy/ReportPolicy.java index bba8b753e..5df1a27bf 100644 --- a/li.strolch.service/src/main/java/li/strolch/report/policy/ReportPolicy.java +++ b/li.strolch.service/src/main/java/li/strolch/report/policy/ReportPolicy.java @@ -6,6 +6,7 @@ import java.util.Set; import java.util.stream.Stream; import com.google.gson.JsonObject; +import li.strolch.model.Resource; import li.strolch.model.StrolchRootElement; import li.strolch.persistence.api.StrolchTransaction; import li.strolch.policy.StrolchPolicy; @@ -27,6 +28,8 @@ public abstract class ReportPolicy extends StrolchPolicy { public abstract boolean hasDateRangeSelector(); + public abstract Resource getReportResource(); + public abstract ReportPolicy dateRange(DateRange dateRange); public abstract List getColumnKeys(); @@ -39,6 +42,8 @@ public abstract class ReportPolicy extends StrolchPolicy { public abstract Stream> buildStream(); + public abstract Stream> buildStream(boolean withOrdering); + public abstract Stream doReport(); public abstract Stream doReportWithPage(int offset, int limit);