[Major] Performance optimizations for reports

The following parameters add optimizations for reports which take a long time to load due to many filter, flat-mapping etc.:

    <Parameter Id="maxFacetValues" Name="Max facet values" Type="Integer" Value="10"/>
    <Parameter Id="maxRowsForFacetGeneration" Name="Max rows for facet generation" Type="Integer" Value="100"/>
    <Parameter Id="directCriteria" Name="Criteria queried directly" Type="StringList" Value="Location"/>

* maxFacetValues -> allows to specify how many facet values are returned to the caller
* maxRowsForFacetGeneration -> specifies after how many seen rows that facet value generation should be stopped
* directCriteria -> allows to define StrolchRootElement types, for which the facet values won't be generated by going through the rows, but are immediately retrieved from the ElementMap. This makes these facets extremely fast, but filtering might not work as expected.

li.strolch.rest/src/main/java/li/strolch/rest/endpoint/ReportResource.java

@@ -2,8 +2,8 @@ package li.strolch.rest.endpoint;
 
 import static java.util.Comparator.comparing;
 import static li.strolch.model.StrolchModelConstants.BAG_PARAMETERS;
-import static li.strolch.model.StrolchModelConstants.TYPE_PARAMETERS;
 import static li.strolch.report.ReportConstants.*;
+import static li.strolch.rest.RestfulStrolchComponent.getInstance;
 import static li.strolch.rest.StrolchRestfulConstants.PARAM_DATE_RANGE_SEL;
 import static li.strolch.rest.StrolchRestfulConstants.*;
 import static li.strolch.utils.helper.StringHelper.*;
@@ -42,7 +42,6 @@ import li.strolch.privilege.model.SimpleRestrictable;
 import li.strolch.report.Report;
 import li.strolch.report.ReportElement;
 import li.strolch.report.ReportSearch;
-import li.strolch.rest.RestfulStrolchComponent;
 import li.strolch.rest.StrolchRestfulConstants;
 import li.strolch.rest.helper.ResponseUtil;
 import li.strolch.utils.ObjectHelper;
@@ -72,9 +71,9 @@ public class ReportResource {
 
 		Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE);
 		if (isEmpty(realm))
-			realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm();
+			realm = getInstance().getContainer().getRealm(cert).getRealm();
 
-		try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext())) {
+		try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext())) {
 
 			StrolchRootElementToJsonVisitor visitor = new StrolchRootElementToJsonVisitor().flat().withoutVersion()
 					.withoutObjectType().withoutPolicies().withoutStateVariables()
@@ -97,7 +96,7 @@ public class ReportResource {
 
 		Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE);
 		if (isEmpty(realm))
-			realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm();
+			realm = getInstance().getContainer().getRealm(cert).getRealm();
 
 		int limit = isNotEmpty(limitS) ? Integer.parseInt(limitS) : 10;
 
@@ -110,8 +109,10 @@ public class ReportResource {
 				localeJ = localesJ.get(cert.getLocale().toLanguageTag()).getAsJsonObject();
 		}
 
-		JsonArray result = new JsonArray();
+		long start = System.nanoTime();
-		try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext());
+
+		JsonObject result = new JsonObject();
+		try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext());
 				Report report = new Report(tx, id)) {
 
 			tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), id));
@@ -120,10 +121,12 @@ public class ReportResource {
 			if (localeJ != null)
 				report.getReportPolicy().setI18nData(localeJ);
 
-			MapOfSets<String, StrolchRootElement> criteria = report.generateFilterCriteria(limit);
+			JsonArray facetsJ = new JsonArray();
-			List<String> types = new ArrayList<>(criteria.keySet());
 			JsonObject finalLocaleJ = localeJ;
-			types.stream().sorted(comparing(type -> {
+
+			MapOfSets<String, StrolchRootElement> criteria = report.generateFilterCriteria(limit);
+
+			criteria.keySet().stream().sorted(comparing(type -> {
 				JsonElement translatedJ = finalLocaleJ == null ? null : finalLocaleJ.get(type);
 				return translatedJ == null ? type : translatedJ.getAsString();
 			})).forEach(type -> {
@@ -136,9 +139,16 @@ public class ReportResource {
 					o.addProperty(Tags.Json.NAME, f.getName());
 					return o;
 				}).collect(JsonArray::new, JsonArray::add, JsonArray::addAll));
-				result.add(filter);
+				facetsJ.add(filter);
 			});
 
+			String duration = formatNanoDuration(System.nanoTime() - start);
+
+			result.add(PARAM_FACETS, facetsJ);
+			result.addProperty(PARAM_DURATION, duration);
+			result.addProperty(PARAM_PARALLEL, report.isParallel());
+
+			logger.info("Facet Generation for " + id + " took: " + duration);
 			return ResponseUtil.toResponse(DATA, result);
 		}
 	}
@@ -153,7 +163,7 @@ public class ReportResource {
 
 		Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE);
 		if (isEmpty(realm))
-			realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm();
+			realm = getInstance().getContainer().getRealm(cert).getRealm();
 
 		String query = isNotEmpty(queryS) ? queryS.toLowerCase() : queryS;
 		int limit = isNotEmpty(limitS) ? Integer.parseInt(limitS) : 10;
@@ -167,7 +177,9 @@ public class ReportResource {
 				localeJ = localesJ.get(cert.getLocale().toLanguageTag()).getAsJsonObject();
 		}
 
-		try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext());
+		long start = System.nanoTime();
+
+		try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext());
 				Report report = new Report(tx, id)) {
 
 			tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), id));
@@ -183,14 +195,31 @@ public class ReportResource {
 				criteria = criteria.filter(f -> ObjectHelper.contains(f.getName(), parts, true));
 			}
 
+			int maxFacetValues;
+			int reportMaxFacetValues = report.getReportResource().getInteger(PARAM_MAX_FACET_VALUES);
+			if (reportMaxFacetValues != 0 && reportMaxFacetValues != limit) {
+				logger.warn("Report " + report.getReportResource().getId() + " has " + PARAM_MAX_FACET_VALUES
+						+ " defined as " + reportMaxFacetValues + ". Ignoring requested limit " + limit);
+				maxFacetValues = reportMaxFacetValues;
+			} else {
+				maxFacetValues = limit;
+			}
+
+			criteria = criteria.sorted(comparing(StrolchElement::getName));
+
+			if (maxFacetValues != 0)
+				criteria = criteria.limit(maxFacetValues);
+
 			// add the data finally
-			JsonArray array = criteria.limit(limit).sorted(comparing(StrolchElement::getName)).map(f -> {
+			JsonArray array = criteria.map(f -> {
 				JsonObject o = new JsonObject();
 				o.addProperty(Tags.Json.ID, f.getId());
 				o.addProperty(Tags.Json.NAME, f.getName());
 				return o;
 			}).collect(JsonArray::new, JsonArray::add, JsonArray::addAll);
 
+			String duration = formatNanoDuration(System.nanoTime() - start);
+			logger.info("Facet Generation for " + id + "." + type + " took: " + duration);
 			return ResponseUtil.toResponse(DATA, array);
 		}
 	}
@@ -204,7 +233,7 @@ public class ReportResource {
 
 		Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE);
 		if (isEmpty(realm))
-			realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm();
+			realm = getInstance().getContainer().getRealm(cert).getRealm();
 
 		DBC.PRE.assertNotEmpty("report ID is required", id);
 
@@ -255,7 +284,7 @@ public class ReportResource {
 
 		long start = System.nanoTime();
 
-		try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext());
+		try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext());
 				Report report = new Report(tx, id)) {
 
 			tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), id));
@@ -337,7 +366,7 @@ public class ReportResource {
 
 		Certificate cert = (Certificate) request.getAttribute(StrolchRestfulConstants.STROLCH_CERTIFICATE);
 		if (isEmpty(realm))
-			realm = RestfulStrolchComponent.getInstance().getContainer().getRealm(cert).getRealm();
+			realm = getInstance().getContainer().getRealm(cert).getRealm();
 
 		DBC.PRE.assertNotEmpty("report ID is required", id);
 
@@ -398,7 +427,7 @@ public class ReportResource {
 
 		return out -> {
 
-			try (StrolchTransaction tx = RestfulStrolchComponent.getInstance().openTx(cert, realm, getContext());
+			try (StrolchTransaction tx = getInstance().openTx(cert, realm, getContext());
 					Report report = new Report(tx, reportId)) {
 
 				tx.getPrivilegeContext().validateAction(new SimpleRestrictable(ReportSearch.class.getName(), reportId));

li.strolch.service/src/main/java/li/strolch/report/Report.java

@@ -35,6 +35,10 @@ public class Report implements AutoCloseable {
 		return this.reportPolicy;
 	}
 
+	public Resource getReportResource() {
+		return this.reportPolicy.getReportResource();
+	}
+
 	public boolean isParallel() {
 		return this.reportPolicy.isParallel();
 	}

li.strolch.service/src/main/java/li/strolch/report/ReportConstants.java

@@ -15,12 +15,14 @@ public class ReportConstants {
 	public static final String PARAM_PARALLEL = "parallel";
 	public static final String PARAM_DESCENDING = "descending";
 	public static final String PARAM_DATE_RANGE_SEL = "dateRangeSel";
+	public static final String PARAM_DIRECT_CRITERIA = "directCriteria";
 	public static final String PARAM_FIELD_REF = "fieldRef";
 	public static final String PARAM_FIELD_REF1 = "fieldRef1";
 	public static final String PARAM_FIELD_REF2 = "fieldRef2";
 	public static final String PARAM_ALLOW_MISSING_COLUMNS = "allowMissingColumns";
 	public static final String PARAM_FILTER_MISSING_VALUES_AS_TRUE = "filterMissingValuesAsTrue";
 	public static final String PARAM_MAX_ROWS_FOR_FACET_GENERATION = "maxRowsForFacetGeneration";
+	public static final String PARAM_MAX_FACET_VALUES = "maxFacetValues";
 	public static final String PARAM_POLICY = "policy";
 
 	public static final String PARAM_JOIN_PARAM = "joinParam";

li.strolch.service/src/main/java/li/strolch/report/policy/GenericReport.java

@@ -40,11 +40,13 @@ import li.strolch.utils.iso8601.ISO8601;
  */
 public class GenericReport extends ReportPolicy {
 
+	public static final int MAX_FACET_VALUE_LIMIT = 100;
 	protected Resource reportRes;
 
 	protected ParameterBag columnsBag;
 	protected List<StringParameter> orderingParams;
 	protected Map<String, StringParameter> filterCriteriaParams;
+	protected Set<String> directCriteria;
 	protected boolean parallel;
 	protected boolean descending;
 	protected boolean allowMissingColumns;
@@ -185,6 +187,14 @@ public class GenericReport extends ReportPolicy {
 
 			this.filtersByPolicy.put(filterFunction, refTuple);
 		}
+
+		// get the list of types of criteria to query directly, not over the element stream
+		this.directCriteria = new HashSet<>(this.reportRes.getStringList(PARAM_DIRECT_CRITERIA));
+	}
+
+	@Override
+	public Resource getReportResource() {
+		return this.reportRes;
 	}
 
 	public boolean isDescending() {
@@ -336,6 +346,17 @@ public class GenericReport extends ReportPolicy {
 	 */
 	@Override
 	public Stream<Map<String, StrolchRootElement>> buildStream() {
+		return buildStream(true);
+	}
+
+	/**
+	 * Builds the stream of rows on which further transformations can be performed. Each row is a {@link Map} for where
+	 * the key is an element type, and the value is the associated element
+	 *
+	 * @return this for chaining
+	 */
+	@Override
+	public Stream<Map<String, StrolchRootElement>> buildStream(boolean withOrdering) {
 
 		Stream<Map<String, StrolchRootElement>> stream;
 
@@ -354,7 +375,7 @@ public class GenericReport extends ReportPolicy {
 
 		stream = stream.peek(e -> incrementCounter());
 
-		if (hasOrdering())
+		if (withOrdering && hasOrdering())
 			stream = stream.sorted(this::sort);
 
 		return stream;
@@ -442,7 +463,7 @@ public class GenericReport extends ReportPolicy {
 						"Additional join type " + joinWithP.getUom() + " is not available on row for "
 								+ joinWithP.getLocator());
 
-			Optional<Parameter<?>> refP = lookupParameter(joinWithP, joinElement);
+			Optional<Parameter<?>> refP = lookupParameter(joinWithP, joinElement, false);
 			if (refP.isEmpty()) {
 				throw new IllegalStateException(
 						"Parameter reference (" + joinWithP.getValue() + ") for " + joinWithP.getLocator()
@@ -517,17 +538,32 @@ public class GenericReport extends ReportPolicy {
 	 * Generates the filter criteria for this report, i.e. it returns a {@link MapOfSets} which defines the type of
 	 * elements on which a filter can be set and the {@link Set} of IDs which can be used for filtering.
 	 *
+	 * @param limit
+	 * 		the max number of values per filter criteria to return
+	 *
 	 * @return the filter criteria as a map of sets
 	 */
 	@Override
 	public MapOfSets<String, StrolchRootElement> generateFilterCriteria(int limit) {
 
+		if (limit <= 0 || limit >= MAX_FACET_VALUE_LIMIT) {
+			logger.warn("Overriding invalid limit " + limit + " with " + MAX_FACET_VALUE_LIMIT);
+			limit = 100;
+		}
+
+		int maxFacetValues;
+		int reportMaxFacetValues = this.reportRes.getInteger(PARAM_MAX_FACET_VALUES);
+		if (reportMaxFacetValues != 0 && reportMaxFacetValues != limit) {
+			logger.warn("Report " + this.reportRes.getId() + " has " + PARAM_MAX_FACET_VALUES + " defined as "
+					+ reportMaxFacetValues + ". Ignoring requested limit " + limit);
+			maxFacetValues = reportMaxFacetValues;
+		} else {
+			maxFacetValues = limit;
+		}
+
 		MapOfSets<String, StrolchRootElement> result = new MapOfSets<>(true);
 
-		Iterator<Map<String, StrolchRootElement>> iter = buildStream().iterator();
+		// we need the list of possible element types, which designate the criteria
-		if (!iter.hasNext())
-			return result;
-
 		List<String> criteria = this.filterCriteriaParams.values().stream() //
 				.filter(p -> {
 					if (p.getUom().equals(UOM_NONE))
@@ -538,34 +574,72 @@ public class GenericReport extends ReportPolicy {
 					return filterCriteriaAllowed(p.getId());
 				}) //
 				.sorted(comparing(StringParameter::getIndex)) //
-				.map(StringParameter::getUom).collect(toList());
+				.map(StringParameter::getUom) //
+				.collect(toList());
+
+		int maxRowsForFacetGeneration = this.reportRes.getInteger(PARAM_MAX_ROWS_FOR_FACET_GENERATION);
+
+		if (!this.directCriteria.isEmpty()) {
+			criteria.forEach(type -> {
+				if (!this.directCriteria.contains(type))
+					return;
+				StringParameter filterCriteriaP = this.filterCriteriaParams.get(type);
+				Stream<? extends StrolchRootElement> stream;
+				switch (filterCriteriaP.getInterpretation()) {
+				case INTERPRETATION_RESOURCE_REF:
+					stream = tx().streamResources(filterCriteriaP.getUom());
+					break;
+				case INTERPRETATION_ORDER_REF:
+					stream = tx().streamOrders(filterCriteriaP.getUom());
+					break;
+				case INTERPRETATION_ACTIVITY_REF:
+					stream = tx().streamActivities(filterCriteriaP.getUom());
+					break;
+				default:
+					throw new IllegalArgumentException("Unhandled element type " + filterCriteriaP.getInterpretation());
+				}
+
+				stream = stream.map(this::mapFilterCriteria).filter(this::filterDirectCriteria);
+
+				if (hasOrdering())
+					stream = stream.sorted(this::sortDirectCriteria);
+
+				if (maxFacetValues > 0)
+					stream = stream.limit(maxFacetValues);
+
+				stream.forEachOrdered(e -> result.addElement(e.getType(), e));
+			});
+
+			criteria.removeAll(this.directCriteria);
+		}
+
+		Stream<Map<String, StrolchRootElement>> stream = buildStream(false);
+		if (maxRowsForFacetGeneration > 0)
+			stream = stream.limit(maxRowsForFacetGeneration);
+
+		Iterator<Map<String, StrolchRootElement>> iter = stream.iterator();
 
-		boolean maxRowsForFacetGeneration = this.reportRes.getBoolean(PARAM_MAX_ROWS_FOR_FACET_GENERATION);
-		long count = 0;
 		while (iter.hasNext()) {
 			Map<String, StrolchRootElement> row = iter.next();
 
 			for (String criterion : criteria) {
-				if (row.containsKey(criterion)) {
+				if (row.containsKey(criterion) && result.size(criterion) < maxFacetValues)
-					if (result.size(criterion) >= limit)
-						continue;
-
 					result.addElement(criterion, row.get(criterion));
-				}
 			}
 
 			// stop if we have enough data
-			count++;
+			if (result.stream().filter(e -> !this.directCriteria.contains(e.getKey()))
-//			if (trimFacetValues) {
+					.mapToInt(e -> e.getValue().size()).allMatch(v -> v >= maxFacetValues))
-//
-//			}
-			if (result.stream().mapToInt(e -> e.getValue().size()).allMatch(v -> v >= limit))
 				break;
 		}
 
 		return result;
 	}
 
+	protected StrolchRootElement mapFilterCriteria(StrolchRootElement element) {
+		return element;
+	}
+
 	@Override
 	public Stream<StrolchRootElement> generateFilterCriteria(String type) {
 		return buildStream().filter(row -> row.containsKey(type)).map(row -> row.get(type)).distinct();
@@ -580,6 +654,56 @@ public class GenericReport extends ReportPolicy {
 		return this.orderingParams != null && !this.orderingParams.isEmpty();
 	}
 
+	protected int sortDirectCriteria(StrolchRootElement column1, StrolchRootElement column2) {
+		if (column1 == null && column2 == null)
+			return 0;
+		if (column1 == null)
+			return -1;
+		if (column2 == null)
+			return 1;
+
+		for (StringParameter fieldRefP : this.orderingParams) {
+			String type = fieldRefP.getUom();
+
+			if (!column1.getType().equals(type))
+				continue;
+
+			int sortVal;
+			if (fieldRefP.getValue().startsWith("$")) {
+				Object columnValue1 = evaluateColumnValue(fieldRefP, Map.of(column1.getType(), column1), false);
+				Object columnValue2 = evaluateColumnValue(fieldRefP, Map.of(column2.getType(), column2), false);
+
+				if (this.descending) {
+					sortVal = ObjectHelper.compare(columnValue2, columnValue1, true);
+				} else {
+					sortVal = ObjectHelper.compare(columnValue1, columnValue2, true);
+				}
+
+			} else {
+				Optional<Parameter<?>> param1 = lookupParameter(fieldRefP, column1, false);
+				Optional<Parameter<?>> param2 = lookupParameter(fieldRefP, column2, false);
+
+				if (param1.isEmpty() && param2.isEmpty())
+					continue;
+
+				if (param1.isPresent() && param2.isEmpty())
+					return 1;
+				else if (param1.isEmpty())
+					return -1;
+
+				if (this.descending)
+					sortVal = param2.get().compareTo(param1.get());
+				else
+					sortVal = param1.get().compareTo(param2.get());
+			}
+
+			if (sortVal != 0)
+				return sortVal;
+		}
+
+		return 0;
+	}
+
 	/**
 	 * Implements a sorting of the given two rows. This implementation using the ordering as is defined in {@link
 	 * ReportConstants#BAG_ORDERING}
@@ -618,8 +742,8 @@ public class GenericReport extends ReportPolicy {
 				}
 
 			} else {
-				Optional<Parameter<?>> param1 = lookupParameter(fieldRefP, column1);
+				Optional<Parameter<?>> param1 = lookupParameter(fieldRefP, column1, false);
-				Optional<Parameter<?>> param2 = lookupParameter(fieldRefP, column2);
+				Optional<Parameter<?>> param2 = lookupParameter(fieldRefP, column2, false);
 
 				if (param1.isEmpty() && param2.isEmpty())
 					continue;
@@ -653,6 +777,39 @@ public class GenericReport extends ReportPolicy {
 				&& !this.filtersById.isEmpty());
 	}
 
+	protected boolean filterDirectCriteria(StrolchRootElement element) {
+
+		// do filtering by policies
+		for (ReportFilterPolicy filterPolicy : this.filtersByPolicy.keySet()) {
+			TypedTuple<StringParameter, StringParameter> refTuple = this.filtersByPolicy.get(filterPolicy);
+			if (refTuple.hasBoth()) {
+				// not applicable for direct criteria
+				continue;
+			}
+
+			// not for this element
+			if (!refTuple.getFirst().getUom().equals(element.getType()))
+				continue;
+
+			Object value = evaluateColumnValue(refTuple.getFirst(), Map.of(element.getType(), element), true);
+			if (this.filterMissingValuesAsTrue && value == null)
+				continue;
+			if (value == null || !filterPolicy.filter(value))
+				return false;
+		}
+
+		// then we do a filter by criteria
+		if (this.filtersById != null && !this.filtersById.isEmpty() && this.filtersById.containsSet(
+				element.getType())) {
+
+			if (!this.filtersById.getSet(element.getType()).contains(element.getId()))
+				return false;
+		}
+
+		// otherwise we want to keep this row
+		return true;
+	}
+
 	/**
 	 * Returns true if the element is filtered, i.e. is to be kep, false if it should not be kept in the stream
 	 *
@@ -667,7 +824,7 @@ public class GenericReport extends ReportPolicy {
 		for (ReportFilterPolicy filterPolicy : this.filtersByPolicy.keySet()) {
 			TypedTuple<StringParameter, StringParameter> refTuple = this.filtersByPolicy.get(filterPolicy);
 
-			if (refTuple.hasFirst() && refTuple.hasSecond()) {
+			if (refTuple.hasBoth()) {
 				Object value1 = evaluateColumnValue(refTuple.getFirst(), row, true);
 				Object value2 = evaluateColumnValue(refTuple.getSecond(), row, true);
 
@@ -702,7 +859,7 @@ public class GenericReport extends ReportPolicy {
 			if (dateRangeSel.equals(COL_DATE)) {
 				date = element.accept(new ElementZdtDateVisitor());
 			} else {
-				Optional<Parameter<?>> param = lookupParameter(this.dateRangeSelP, element);
+				Optional<Parameter<?>> param = lookupParameter(this.dateRangeSelP, element, false);
 				if (param.isEmpty() || param.get().getValueType() != StrolchValueType.DATE)
 					throw new IllegalStateException(
 							"Date Range selector is invalid, as referenced parameter is not a Date but "
@@ -776,7 +933,8 @@ public class GenericReport extends ReportPolicy {
 			else
 				columnValue = parameter;
 		} else {
-			columnValue = lookupParameter(columnDefP, column).orElseGet(() -> allowNull ? null : new StringParameter());
+			columnValue = lookupParameter(columnDefP, column, allowNull) //
+					.orElseGet(() -> allowNull ? null : new StringParameter());
 		}
 
 		return columnValue;
@@ -828,7 +986,8 @@ public class GenericReport extends ReportPolicy {
 	 *
 	 * @return the {@link Optional} with the parameter
 	 */
-	protected Optional<Parameter<?>> lookupParameter(StringParameter paramRefP, StrolchRootElement element) {
+	protected Optional<Parameter<?>> lookupParameter(StringParameter paramRefP, StrolchRootElement element,
+			boolean overrideAllowMissingColumns) {
 		String paramRef = paramRefP.getValue();
 
 		String[] locatorParts = paramRef.split(Locator.PATH_SEPARATOR);
@@ -841,7 +1000,7 @@ public class GenericReport extends ReportPolicy {
 		String paramKey = locatorParts[2];
 
 		Parameter<?> param = element.getParameter(bagKey, paramKey);
-		if (!allowMissingColumns && param == null)
+		if (!overrideAllowMissingColumns && !this.allowMissingColumns && param == null)
 			throw new IllegalStateException(
 					"Parameter reference (" + paramRef + ") for " + paramRefP.getLocator() + " not found on "
 							+ element.getLocator());

li.strolch.service/src/main/java/li/strolch/report/policy/ReportPolicy.java

@@ -6,6 +6,7 @@ import java.util.Set;
 import java.util.stream.Stream;
 
 import com.google.gson.JsonObject;
+import li.strolch.model.Resource;
 import li.strolch.model.StrolchRootElement;
 import li.strolch.persistence.api.StrolchTransaction;
 import li.strolch.policy.StrolchPolicy;
@@ -27,6 +28,8 @@ public abstract class ReportPolicy extends StrolchPolicy {
 
 	public abstract boolean hasDateRangeSelector();
 
+	public abstract Resource getReportResource();
+
 	public abstract ReportPolicy dateRange(DateRange dateRange);
 
 	public abstract List<String> getColumnKeys();
@@ -39,6 +42,8 @@ public abstract class ReportPolicy extends StrolchPolicy {
 
 	public abstract Stream<Map<String, StrolchRootElement>> buildStream();
 
+	public abstract Stream<Map<String, StrolchRootElement>> buildStream(boolean withOrdering);
+
 	public abstract Stream<ReportElement> doReport();
 
 	public abstract Stream<ReportElement> doReportWithPage(int offset, int limit);