1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.syncope.core.persistence.jpa.dao;
20
21 import co.elastic.clients.elasticsearch.ElasticsearchClient;
22 import co.elastic.clients.elasticsearch._types.FieldSort;
23 import co.elastic.clients.elasticsearch._types.FieldValue;
24 import co.elastic.clients.elasticsearch._types.SearchType;
25 import co.elastic.clients.elasticsearch._types.SortOptions;
26 import co.elastic.clients.elasticsearch._types.SortOrder;
27 import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
28 import co.elastic.clients.elasticsearch._types.query_dsl.DisMaxQuery;
29 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
30 import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders;
31 import co.elastic.clients.elasticsearch.core.CountRequest;
32 import co.elastic.clients.elasticsearch.core.SearchRequest;
33 import co.elastic.clients.elasticsearch.core.search.Hit;
34 import co.elastic.clients.json.JsonData;
35 import java.lang.reflect.Field;
36 import java.util.ArrayList;
37 import java.util.HashSet;
38 import java.util.List;
39 import java.util.Optional;
40 import java.util.Set;
41 import java.util.stream.Collectors;
42 import org.apache.commons.lang3.tuple.Pair;
43 import org.apache.commons.lang3.tuple.Triple;
44 import org.apache.syncope.common.lib.SyncopeClientException;
45 import org.apache.syncope.common.lib.SyncopeConstants;
46 import org.apache.syncope.common.lib.types.AnyTypeKind;
47 import org.apache.syncope.common.lib.types.AttrSchemaType;
48 import org.apache.syncope.common.lib.types.ClientExceptionType;
49 import org.apache.syncope.common.rest.api.service.JAXRSService;
50 import org.apache.syncope.core.persistence.api.attrvalue.validation.PlainAttrValidationManager;
51 import org.apache.syncope.core.persistence.api.dao.AnyObjectDAO;
52 import org.apache.syncope.core.persistence.api.dao.DynRealmDAO;
53 import org.apache.syncope.core.persistence.api.dao.GroupDAO;
54 import org.apache.syncope.core.persistence.api.dao.PlainSchemaDAO;
55 import org.apache.syncope.core.persistence.api.dao.RealmDAO;
56 import org.apache.syncope.core.persistence.api.dao.UserDAO;
57 import org.apache.syncope.core.persistence.api.dao.search.AnyCond;
58 import org.apache.syncope.core.persistence.api.dao.search.AnyTypeCond;
59 import org.apache.syncope.core.persistence.api.dao.search.AttrCond;
60 import org.apache.syncope.core.persistence.api.dao.search.AuxClassCond;
61 import org.apache.syncope.core.persistence.api.dao.search.DynRealmCond;
62 import org.apache.syncope.core.persistence.api.dao.search.MemberCond;
63 import org.apache.syncope.core.persistence.api.dao.search.MembershipCond;
64 import org.apache.syncope.core.persistence.api.dao.search.OrderByClause;
65 import org.apache.syncope.core.persistence.api.dao.search.PrivilegeCond;
66 import org.apache.syncope.core.persistence.api.dao.search.RelationshipCond;
67 import org.apache.syncope.core.persistence.api.dao.search.RelationshipTypeCond;
68 import org.apache.syncope.core.persistence.api.dao.search.ResourceCond;
69 import org.apache.syncope.core.persistence.api.dao.search.RoleCond;
70 import org.apache.syncope.core.persistence.api.dao.search.SearchCond;
71 import org.apache.syncope.core.persistence.api.entity.Any;
72 import org.apache.syncope.core.persistence.api.entity.AnyUtils;
73 import org.apache.syncope.core.persistence.api.entity.AnyUtilsFactory;
74 import org.apache.syncope.core.persistence.api.entity.DynRealm;
75 import org.apache.syncope.core.persistence.api.entity.EntityFactory;
76 import org.apache.syncope.core.persistence.api.entity.PlainAttrValue;
77 import org.apache.syncope.core.persistence.api.entity.PlainSchema;
78 import org.apache.syncope.core.persistence.api.entity.Realm;
79 import org.apache.syncope.core.provisioning.api.utils.FormatUtils;
80 import org.apache.syncope.core.provisioning.api.utils.RealmUtils;
81 import org.apache.syncope.core.spring.security.AuthContextUtils;
82 import org.apache.syncope.ext.elasticsearch.client.ElasticsearchUtils;
83 import org.springframework.util.CollectionUtils;
84
85
86
87
88 public class ElasticsearchAnySearchDAO extends AbstractAnySearchDAO {
89
90 protected final ElasticsearchClient client;
91
92 protected final int indexMaxResultWindow;
93
94 public ElasticsearchAnySearchDAO(
95 final RealmDAO realmDAO,
96 final DynRealmDAO dynRealmDAO,
97 final UserDAO userDAO,
98 final GroupDAO groupDAO,
99 final AnyObjectDAO anyObjectDAO,
100 final PlainSchemaDAO schemaDAO,
101 final EntityFactory entityFactory,
102 final AnyUtilsFactory anyUtilsFactory,
103 final PlainAttrValidationManager validator,
104 final ElasticsearchClient client,
105 final int indexMaxResultWindow) {
106
107 super(
108 realmDAO,
109 dynRealmDAO,
110 userDAO,
111 groupDAO,
112 anyObjectDAO,
113 schemaDAO,
114 entityFactory,
115 anyUtilsFactory,
116 validator);
117
118 this.client = client;
119 this.indexMaxResultWindow = indexMaxResultWindow;
120 }
121
122 protected Triple<Optional<Query>, Set<String>, Set<String>> getAdminRealmsFilter(
123 final Realm base,
124 final boolean recursive,
125 final Set<String> adminRealms,
126 final AnyTypeKind kind) {
127
128 Set<String> dynRealmKeys = new HashSet<>();
129 Set<String> groupOwners = new HashSet<>();
130 List<Query> queries = new ArrayList<>();
131
132 if (recursive) {
133 adminRealms.forEach(realmPath -> {
134 Optional<Pair<String, String>> goRealm = RealmUtils.parseGroupOwnerRealm(realmPath);
135 if (goRealm.isPresent()) {
136 groupOwners.add(goRealm.get().getRight());
137 } else if (realmPath.startsWith("/")) {
138 Realm realm = Optional.ofNullable(realmDAO.findByFullPath(realmPath)).orElseThrow(() -> {
139 SyncopeClientException noRealm = SyncopeClientException.build(ClientExceptionType.InvalidRealm);
140 noRealm.getElements().add("Invalid realm specified: " + realmPath);
141 return noRealm;
142 });
143
144 realmDAO.findDescendants(realm.getFullPath(), base.getFullPath()).
145 forEach(descendant -> queries.add(
146 new Query.Builder().term(QueryBuilders.term().
147 field("realm").value(descendant).build()).
148 build()));
149 } else {
150 DynRealm dynRealm = dynRealmDAO.find(realmPath);
151 if (dynRealm == null) {
152 LOG.warn("Ignoring invalid dynamic realm {}", realmPath);
153 } else {
154 dynRealmKeys.add(dynRealm.getKey());
155 queries.add(new Query.Builder().term(QueryBuilders.term().
156 field("dynRealm").value(dynRealm.getKey()).build()).
157 build());
158 }
159 }
160 });
161 } else {
162 if (adminRealms.stream().anyMatch(r -> r.startsWith(base.getFullPath()))) {
163 queries.add(new Query.Builder().term(QueryBuilders.term().
164 field("realm").value(base.getKey()).build()).
165 build());
166 }
167 }
168
169 return Triple.of(
170 dynRealmKeys.isEmpty() && groupOwners.isEmpty()
171 ? Optional.of(new Query.Builder().disMax(QueryBuilders.disMax().queries(queries).build()).build())
172 : Optional.empty(),
173 dynRealmKeys,
174 groupOwners);
175 }
176
177 protected Query getQuery(
178 final Realm base,
179 final boolean recursive,
180 final Set<String> adminRealms,
181 final SearchCond cond,
182 final AnyTypeKind kind) {
183
184 Query query;
185 if (SyncopeConstants.FULL_ADMIN_REALMS.equals(adminRealms)) {
186 query = getQuery(cond, kind);
187
188 if (!recursive) {
189 query = new Query.Builder().bool(
190 QueryBuilders.bool().
191 must(new Query.Builder().term(QueryBuilders.term().
192 field("realm").value(base.getKey()).build()).
193 build()).
194 must(query).build()).
195 build();
196 }
197 } else {
198 Triple<Optional<Query>, Set<String>, Set<String>> filter =
199 getAdminRealmsFilter(base, recursive, adminRealms, kind);
200 query = getQuery(buildEffectiveCond(cond, filter.getMiddle(), filter.getRight(), kind), kind);
201
202 if (filter.getLeft().isPresent()) {
203 query = new Query.Builder().bool(
204 QueryBuilders.bool().
205 must(filter.getLeft().get()).
206 must(query).build()).
207 build();
208 }
209 }
210
211 return query;
212 }
213
214 @Override
215 protected int doCount(
216 final Realm base,
217 final boolean recursive,
218 final Set<String> adminRealms,
219 final SearchCond cond,
220 final AnyTypeKind kind) {
221
222 CountRequest request = new CountRequest.Builder().
223 index(ElasticsearchUtils.getAnyIndex(AuthContextUtils.getDomain(), kind)).
224 query(getQuery(base, recursive, adminRealms, cond, kind)).
225 build();
226 LOG.debug("Count JSON request: {}", request);
227
228 try {
229 return (int) client.count(request).count();
230 } catch (Exception e) {
231 LOG.error("While counting in Elasticsearch", e);
232 return 0;
233 }
234 }
235
236 protected List<SortOptions> sortBuilders(
237 final AnyTypeKind kind,
238 final List<OrderByClause> orderBy) {
239
240 AnyUtils anyUtils = anyUtilsFactory.getInstance(kind);
241
242 List<SortOptions> options = new ArrayList<>();
243 orderBy.forEach(clause -> {
244 String sortName = null;
245
246
247 String fieldName = "key".equals(clause.getField()) ? "id" : clause.getField();
248
249 Field anyField = anyUtils.getField(fieldName);
250 if (anyField == null) {
251 PlainSchema schema = plainSchemaDAO.find(fieldName);
252 if (schema != null) {
253 sortName = fieldName;
254 }
255 } else {
256 sortName = fieldName;
257 }
258
259 if (sortName == null) {
260 LOG.warn("Cannot build any valid clause from {}", clause);
261 } else {
262 options.add(new SortOptions.Builder().field(
263 new FieldSort.Builder().
264 field(sortName).
265 order(clause.getDirection() == OrderByClause.Direction.ASC
266 ? SortOrder.Asc : SortOrder.Desc).
267 build()).
268 build());
269 }
270 });
271 return options;
272 }
273
274 @Override
275 protected <T extends Any<?>> List<T> doSearch(
276 final Realm base,
277 final boolean recursive,
278 final Set<String> adminRealms,
279 final SearchCond cond,
280 final int page,
281 final int itemsPerPage,
282 final List<OrderByClause> orderBy,
283 final AnyTypeKind kind) {
284
285 SearchRequest request = new SearchRequest.Builder().
286 index(ElasticsearchUtils.getAnyIndex(AuthContextUtils.getDomain(), kind)).
287 searchType(SearchType.QueryThenFetch).
288 query(getQuery(base, recursive, adminRealms, cond, kind)).
289 from(itemsPerPage * (page <= 0 ? 0 : page - 1)).
290 size(itemsPerPage < 0 ? indexMaxResultWindow : itemsPerPage).
291 sort(sortBuilders(kind, orderBy)).
292 build();
293 LOG.debug("Search JSON request: {}", request);
294
295 List<Hit<Void>> esResult = null;
296 try {
297 esResult = client.search(request, Void.class).hits().hits();
298 } catch (Exception e) {
299 LOG.error("While searching in Elasticsearch", e);
300 }
301
302 return CollectionUtils.isEmpty(esResult)
303 ? List.of()
304 : buildResult(esResult.stream().map(Hit::id).collect(Collectors.toList()), kind);
305 }
306
307 protected Query getQuery(final SearchCond cond, final AnyTypeKind kind) {
308 Query query = null;
309
310 switch (cond.getType()) {
311 case LEAF:
312 case NOT_LEAF:
313 query = cond.getLeaf(AnyTypeCond.class).
314 filter(leaf -> AnyTypeKind.ANY_OBJECT == kind).
315 map(this::getQuery).
316 orElse(null);
317
318 if (query == null) {
319 query = cond.getLeaf(RelationshipTypeCond.class).
320 filter(leaf -> AnyTypeKind.GROUP != kind).
321 map(this::getQuery).
322 orElse(null);
323 }
324
325 if (query == null) {
326 query = cond.getLeaf(RelationshipCond.class).
327 filter(leaf -> AnyTypeKind.GROUP != kind).
328 map(this::getQuery).
329 orElse(null);
330 }
331
332 if (query == null) {
333 query = cond.getLeaf(MembershipCond.class).
334 filter(leaf -> AnyTypeKind.GROUP != kind).
335 map(this::getQuery).
336 orElse(null);
337 }
338
339 if (query == null) {
340 query = cond.getLeaf(MemberCond.class).
341 filter(leaf -> AnyTypeKind.GROUP == kind).
342 map(this::getQuery).
343 orElse(null);
344 }
345
346 if (query == null) {
347 query = cond.getLeaf(RoleCond.class).
348 filter(leaf -> AnyTypeKind.USER == kind).
349 map(this::getQuery).
350 orElse(null);
351 }
352
353 if (query == null) {
354 query = cond.getLeaf(PrivilegeCond.class).
355 filter(leaf -> AnyTypeKind.USER == kind).
356 map(this::getQuery).
357 orElse(null);
358 }
359
360 if (query == null) {
361 query = cond.getLeaf(DynRealmCond.class).
362 map(this::getQuery).
363 orElse(null);
364 }
365
366 if (query == null) {
367 query = cond.getLeaf(AuxClassCond.class).
368 map(this::getQuery).
369 orElse(null);
370 }
371
372 if (query == null) {
373 query = cond.getLeaf(ResourceCond.class).
374 map(this::getQuery).
375 orElse(null);
376 }
377
378 if (query == null) {
379 query = cond.getLeaf(AnyCond.class).map(ac -> getQuery(ac, kind)).
380 or(() -> cond.getLeaf(AttrCond.class).map(ac -> getQuery(ac, kind))).
381 orElse(null);
382 }
383
384
385 if (query == null) {
386 query = getQueryForCustomConds(cond, kind);
387 }
388
389 if (query == null) {
390 throw new IllegalArgumentException("Cannot construct QueryBuilder");
391 }
392
393 if (cond.getType() == SearchCond.Type.NOT_LEAF) {
394 query = new Query.Builder().bool(QueryBuilders.bool().mustNot(query).build()).build();
395 }
396 break;
397
398 case AND:
399 List<Query> andCompound = new ArrayList<>();
400
401 Query andLeft = getQuery(cond.getLeft(), kind);
402 if (andLeft._kind() == Query.Kind.Bool && !((BoolQuery) andLeft._get()).must().isEmpty()) {
403 andCompound.addAll(((BoolQuery) andLeft._get()).must());
404 } else {
405 andCompound.add(andLeft);
406 }
407
408 Query andRight = getQuery(cond.getRight(), kind);
409 if (andRight._kind() == Query.Kind.Bool && !((BoolQuery) andRight._get()).must().isEmpty()) {
410 andCompound.addAll(((BoolQuery) andRight._get()).must());
411 } else {
412 andCompound.add(andRight);
413 }
414
415 query = new Query.Builder().bool(QueryBuilders.bool().must(andCompound).build()).build();
416 break;
417
418 case OR:
419 List<Query> orCompound = new ArrayList<>();
420
421 Query orLeft = getQuery(cond.getLeft(), kind);
422 if (orLeft._kind() == Query.Kind.DisMax) {
423 orCompound.addAll(((DisMaxQuery) orLeft._get()).queries());
424 } else {
425 orCompound.add(orLeft);
426 }
427
428 Query orRight = getQuery(cond.getRight(), kind);
429 if (orRight._kind() == Query.Kind.DisMax) {
430 orCompound.addAll(((DisMaxQuery) orRight._get()).queries());
431 } else {
432 orCompound.add(orRight);
433 }
434
435 query = new Query.Builder().disMax(QueryBuilders.disMax().queries(orCompound).build()).build();
436 break;
437
438 default:
439 }
440
441 return query;
442 }
443
444 protected Query getQuery(final AnyTypeCond cond) {
445 return new Query.Builder().term(QueryBuilders.term().
446 field("anyType").value(cond.getAnyTypeKey()).build()).
447 build();
448 }
449
450 protected Query getQuery(final RelationshipTypeCond cond) {
451 return new Query.Builder().term(QueryBuilders.term().
452 field("relationshipTypes").value(cond.getRelationshipTypeKey()).build()).
453 build();
454 }
455
456 protected Query getQuery(final RelationshipCond cond) {
457 List<Query> queries = check(cond).stream().
458 map(key -> new Query.Builder().term(QueryBuilders.term().
459 field("relationships").value(key).build()).
460 build()).collect(Collectors.toList());
461
462 return queries.size() == 1
463 ? queries.get(0)
464 : new Query.Builder().disMax(QueryBuilders.disMax().queries(queries).build()).build();
465 }
466
467 protected Query getQuery(final MembershipCond cond) {
468 List<Query> queries = check(cond).stream().
469 map(key -> new Query.Builder().term(QueryBuilders.term().
470 field("memberships").value(key).build()).
471 build()).collect(Collectors.toList());
472
473 return queries.size() == 1
474 ? queries.get(0)
475 : new Query.Builder().disMax(QueryBuilders.disMax().queries(queries).build()).build();
476 }
477
478 protected Query getQuery(final RoleCond cond) {
479 return new Query.Builder().term(QueryBuilders.term().
480 field("roles").value(cond.getRole()).build()).
481 build();
482 }
483
484 protected Query getQuery(final PrivilegeCond cond) {
485 return new Query.Builder().term(QueryBuilders.term().
486 field("privileges").value(cond.getPrivilege()).build()).
487 build();
488 }
489
490 protected Query getQuery(final DynRealmCond cond) {
491 return new Query.Builder().term(QueryBuilders.term().
492 field("dynRealms").value(cond.getDynRealm()).build()).
493 build();
494 }
495
496 protected Query getQuery(final MemberCond cond) {
497 List<Query> queries = check(cond).stream().
498 map(key -> new Query.Builder().term(QueryBuilders.term().
499 field("members").value(key).build()).
500 build()).collect(Collectors.toList());
501
502 return queries.size() == 1
503 ? queries.get(0)
504 : new Query.Builder().disMax(QueryBuilders.disMax().queries(queries).build()).build();
505 }
506
507 protected Query getQuery(final AuxClassCond cond) {
508 return new Query.Builder().term(QueryBuilders.term().
509 field("auxClasses").value(cond.getAuxClass()).build()).
510 build();
511 }
512
513 protected Query getQuery(final ResourceCond cond) {
514 return new Query.Builder().term(QueryBuilders.term().
515 field("resources").value(cond.getResource()).build()).
516 build();
517 }
518
519 protected Query fillAttrQuery(
520 final PlainSchema schema,
521 final PlainAttrValue attrValue,
522 final AttrCond cond) {
523
524 Object value = schema.getType() == AttrSchemaType.Date && attrValue.getDateValue() != null
525 ? FormatUtils.format(attrValue.getDateValue())
526 : attrValue.getValue();
527
528 Query query = null;
529
530 switch (cond.getType()) {
531 case ISNOTNULL:
532 query = new Query.Builder().exists(QueryBuilders.exists().field(schema.getKey()).build()).build();
533 break;
534
535 case ISNULL:
536 query = new Query.Builder().bool(QueryBuilders.bool().mustNot(
537 new Query.Builder().exists(QueryBuilders.exists().field(schema.getKey()).build()).build()).
538 build()).build();
539 break;
540
541 case ILIKE:
542 StringBuilder output = new StringBuilder();
543 for (char c : cond.getExpression().toLowerCase().toCharArray()) {
544 if (c == '%') {
545 output.append(".*");
546 } else if (Character.isLetter(c)) {
547 output.append('[').
548 append(c).
549 append(Character.toUpperCase(c)).
550 append(']');
551 } else {
552 output.append(ElasticsearchUtils.escapeForLikeRegex(c));
553 }
554 }
555 query = new Query.Builder().regexp(QueryBuilders.regexp().
556 field(schema.getKey()).value(output.toString()).build()).build();
557 break;
558
559 case LIKE:
560 query = new Query.Builder().wildcard(QueryBuilders.wildcard().
561 field(schema.getKey()).value(cond.getExpression().replace('%', '*')).build()).build();
562 break;
563
564 case IEQ:
565 query = new Query.Builder().match(QueryBuilders.match().
566 field(schema.getKey()).query(cond.getExpression().toLowerCase()).build()).
567 build();
568 break;
569
570 case EQ:
571 FieldValue fieldValue;
572 if (value instanceof Double) {
573 fieldValue = FieldValue.of((Double) value);
574 } else if (value instanceof Long) {
575 fieldValue = FieldValue.of((Long) value);
576 } else if (value instanceof Boolean) {
577 fieldValue = FieldValue.of((Boolean) value);
578 } else {
579 fieldValue = FieldValue.of(value.toString());
580 }
581 query = new Query.Builder().term(QueryBuilders.term().
582 field(schema.getKey()).value(fieldValue).build()).
583 build();
584 break;
585
586 case GE:
587 query = new Query.Builder().range(QueryBuilders.range().
588 field(schema.getKey()).gte(JsonData.of(value)).build()).
589 build();
590 break;
591
592 case GT:
593 query = new Query.Builder().range(QueryBuilders.range().
594 field(schema.getKey()).gt(JsonData.of(value)).build()).
595 build();
596 break;
597
598 case LE:
599 query = new Query.Builder().range(QueryBuilders.range().
600 field(schema.getKey()).lte(JsonData.of(value)).build()).
601 build();
602 break;
603
604 case LT:
605 query = new Query.Builder().range(QueryBuilders.range().
606 field(schema.getKey()).lt(JsonData.of(value)).build()).
607 build();
608 break;
609
610 default:
611 }
612
613 return query;
614 }
615
616 protected Query getQuery(final AttrCond cond, final AnyTypeKind kind) {
617 Pair<PlainSchema, PlainAttrValue> checked = check(cond, kind);
618
619 return fillAttrQuery(checked.getLeft(), checked.getRight(), cond);
620 }
621
622 protected Query getQuery(final AnyCond cond, final AnyTypeKind kind) {
623 if (JAXRSService.PARAM_REALM.equals(cond.getSchema()) && cond.getExpression().startsWith("/")) {
624 Realm realm = Optional.ofNullable(realmDAO.findByFullPath(cond.getExpression())).
625 orElseThrow(() -> new IllegalArgumentException("Invalid Realm full path: " + cond.getExpression()));
626 cond.setExpression(realm.getKey());
627 }
628
629 Triple<PlainSchema, PlainAttrValue, AnyCond> checked = check(cond, kind);
630
631 return fillAttrQuery(checked.getLeft(), checked.getMiddle(), checked.getRight());
632 }
633
634 protected Query getQueryForCustomConds(final SearchCond cond, final AnyTypeKind kind) {
635 return null;
636 }
637 }