View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.syncope.fit.core;
20  
21  import static org.awaitility.Awaitility.await;
22  import static org.junit.jupiter.api.Assertions.assertEquals;
23  import static org.junit.jupiter.api.Assertions.assertFalse;
24  import static org.junit.jupiter.api.Assertions.assertNotEquals;
25  import static org.junit.jupiter.api.Assertions.assertNotNull;
26  import static org.junit.jupiter.api.Assertions.assertNull;
27  import static org.junit.jupiter.api.Assertions.assertTrue;
28  import static org.junit.jupiter.api.Assertions.fail;
29  import static org.junit.jupiter.api.Assumptions.assumeFalse;
30  
31  import java.io.FileInputStream;
32  import java.io.FileOutputStream;
33  import java.io.IOException;
34  import java.io.InputStream;
35  import java.io.OutputStream;
36  import java.nio.charset.StandardCharsets;
37  import java.time.OffsetDateTime;
38  import java.util.Collections;
39  import java.util.Date;
40  import java.util.HashSet;
41  import java.util.List;
42  import java.util.Locale;
43  import java.util.Map;
44  import java.util.Optional;
45  import java.util.Properties;
46  import java.util.Set;
47  import java.util.UUID;
48  import java.util.concurrent.ExecutorService;
49  import java.util.concurrent.Executors;
50  import java.util.concurrent.TimeUnit;
51  import java.util.concurrent.atomic.AtomicReference;
52  import javax.naming.NamingException;
53  import javax.naming.directory.Attribute;
54  import javax.naming.directory.BasicAttribute;
55  import javax.ws.rs.core.Response;
56  import org.apache.commons.io.IOUtils;
57  import org.apache.commons.lang3.SerializationUtils;
58  import org.apache.commons.lang3.StringUtils;
59  import org.apache.commons.lang3.tuple.Pair;
60  import org.apache.commons.lang3.tuple.Triple;
61  import org.apache.syncope.client.lib.SyncopeClient;
62  import org.apache.syncope.client.lib.batch.BatchRequest;
63  import org.apache.syncope.common.lib.Attr;
64  import org.apache.syncope.common.lib.SyncopeClientException;
65  import org.apache.syncope.common.lib.SyncopeConstants;
66  import org.apache.syncope.common.lib.policy.PullPolicyTO;
67  import org.apache.syncope.common.lib.request.AnyCR;
68  import org.apache.syncope.common.lib.request.AnyObjectCR;
69  import org.apache.syncope.common.lib.request.GroupCR;
70  import org.apache.syncope.common.lib.request.PasswordPatch;
71  import org.apache.syncope.common.lib.request.ResourceDR;
72  import org.apache.syncope.common.lib.request.UserCR;
73  import org.apache.syncope.common.lib.request.UserUR;
74  import org.apache.syncope.common.lib.to.AnyObjectTO;
75  import org.apache.syncope.common.lib.to.ConnInstanceTO;
76  import org.apache.syncope.common.lib.to.ConnObject;
77  import org.apache.syncope.common.lib.to.ExecTO;
78  import org.apache.syncope.common.lib.to.GroupTO;
79  import org.apache.syncope.common.lib.to.ImplementationTO;
80  import org.apache.syncope.common.lib.to.Item;
81  import org.apache.syncope.common.lib.to.MembershipTO;
82  import org.apache.syncope.common.lib.to.PagedResult;
83  import org.apache.syncope.common.lib.to.Provision;
84  import org.apache.syncope.common.lib.to.ProvisioningResult;
85  import org.apache.syncope.common.lib.to.PullTaskTO;
86  import org.apache.syncope.common.lib.to.RemediationTO;
87  import org.apache.syncope.common.lib.to.ResourceTO;
88  import org.apache.syncope.common.lib.to.TaskTO;
89  import org.apache.syncope.common.lib.to.UserTO;
90  import org.apache.syncope.common.lib.types.AnyTypeKind;
91  import org.apache.syncope.common.lib.types.CipherAlgorithm;
92  import org.apache.syncope.common.lib.types.ClientExceptionType;
93  import org.apache.syncope.common.lib.types.ConnConfProperty;
94  import org.apache.syncope.common.lib.types.ConnectorCapability;
95  import org.apache.syncope.common.lib.types.ExecStatus;
96  import org.apache.syncope.common.lib.types.IdMImplementationType;
97  import org.apache.syncope.common.lib.types.IdRepoImplementationType;
98  import org.apache.syncope.common.lib.types.ImplementationEngine;
99  import org.apache.syncope.common.lib.types.MatchingRule;
100 import org.apache.syncope.common.lib.types.PolicyType;
101 import org.apache.syncope.common.lib.types.PullMode;
102 import org.apache.syncope.common.lib.types.ResourceDeassociationAction;
103 import org.apache.syncope.common.lib.types.ResourceOperation;
104 import org.apache.syncope.common.lib.types.TaskType;
105 import org.apache.syncope.common.lib.types.ThreadPoolSettings;
106 import org.apache.syncope.common.lib.types.UnmatchingRule;
107 import org.apache.syncope.common.rest.api.RESTHeaders;
108 import org.apache.syncope.common.rest.api.beans.AnyQuery;
109 import org.apache.syncope.common.rest.api.beans.ReconQuery;
110 import org.apache.syncope.common.rest.api.beans.RemediationQuery;
111 import org.apache.syncope.common.rest.api.beans.TaskQuery;
112 import org.apache.syncope.common.rest.api.service.ConnectorService;
113 import org.apache.syncope.common.rest.api.service.TaskService;
114 import org.apache.syncope.common.rest.api.service.UserService;
115 import org.apache.syncope.core.provisioning.java.pushpull.DBPasswordPullActions;
116 import org.apache.syncope.core.provisioning.java.pushpull.LDAPPasswordPullActions;
117 import org.apache.syncope.core.spring.security.Encryptor;
118 import org.apache.syncope.fit.core.reference.TestPullActions;
119 import org.identityconnectors.framework.common.objects.Name;
120 import org.junit.jupiter.api.BeforeAll;
121 import org.junit.jupiter.api.Test;
122 import org.springframework.jdbc.core.JdbcTemplate;
123 
124 public class PullTaskITCase extends AbstractTaskITCase {
125 
126     @BeforeAll
127     public static void testPullActionsSetup() {
128         ImplementationTO pullActions = null;
129         try {
130             pullActions = IMPLEMENTATION_SERVICE.read(
131                     IdMImplementationType.PULL_ACTIONS, TestPullActions.class.getSimpleName());
132         } catch (SyncopeClientException e) {
133             if (e.getType().getResponseStatus() == Response.Status.NOT_FOUND) {
134                 pullActions = new ImplementationTO();
135                 pullActions.setKey(TestPullActions.class.getSimpleName());
136                 pullActions.setEngine(ImplementationEngine.JAVA);
137                 pullActions.setType(IdMImplementationType.PULL_ACTIONS);
138                 pullActions.setBody(TestPullActions.class.getName());
139                 Response response = IMPLEMENTATION_SERVICE.create(pullActions);
140                 pullActions = IMPLEMENTATION_SERVICE.read(
141                         pullActions.getType(), response.getHeaderString(RESTHeaders.RESOURCE_KEY));
142                 assertNotNull(pullActions);
143             }
144         }
145         assertNotNull(pullActions);
146 
147         PullTaskTO pullTask = TASK_SERVICE.read(TaskType.PULL, PULL_TASK_KEY, true);
148         pullTask.getActions().add(pullActions.getKey());
149         TASK_SERVICE.update(TaskType.PULL, pullTask);
150     }
151 
152     private static Pair<String, Set<Attribute>> prepareLdapAttributes(
153             final String uid,
154             final String email,
155             final String description,
156             final String givenName,
157             final String sn,
158             final String registeredAddress,
159             final String title,
160             final String password) {
161 
162         String entryDn = "uid=" + uid + ",ou=People,o=isp";
163         Set<Attribute> attributes = new HashSet<>();
164 
165         attributes.add(new BasicAttribute("description", description));
166         attributes.add(new BasicAttribute("givenName", givenName));
167         attributes.add(new BasicAttribute("mail", email));
168         attributes.add(new BasicAttribute("sn", sn));
169         attributes.add(new BasicAttribute("cn", uid));
170         attributes.add(new BasicAttribute("uid", uid));
171         attributes.add(new BasicAttribute("registeredaddress", registeredAddress));
172         attributes.add(new BasicAttribute("title", title));
173         attributes.add(new BasicAttribute("userpassword", password));
174 
175         Attribute oc = new BasicAttribute("objectClass");
176         oc.add("top");
177         oc.add("person");
178         oc.add("inetOrgPerson");
179         oc.add("organizationalPerson");
180         attributes.add(oc);
181 
182         return Pair.of(entryDn, attributes);
183     }
184 
185     @Test
186     public void getPullActionsClasses() {
187         Set<String> actions = ANONYMOUS_CLIENT.platform().
188                 getJavaImplInfo(IdMImplementationType.PULL_ACTIONS).get().getClasses();
189         assertNotNull(actions);
190         assertFalse(actions.isEmpty());
191     }
192 
193     @Test
194     public void list() {
195         PagedResult<PullTaskTO> tasks = TASK_SERVICE.search(new TaskQuery.Builder(TaskType.PULL).build());
196         assertFalse(tasks.getResult().isEmpty());
197         tasks.getResult().stream().
198                 filter(task -> (!(task instanceof PullTaskTO))).
199                 forEach(item -> fail("This should not happen"));
200     }
201 
202     @Test
203     public void create() {
204         PullTaskTO task = new PullTaskTO();
205         task.setName("Test create Pull");
206         task.setDestinationRealm("/");
207         task.setResource(RESOURCE_NAME_WS2);
208         task.setPullMode(PullMode.FULL_RECONCILIATION);
209 
210         UserTO userTemplate = new UserTO();
211         userTemplate.getResources().add(RESOURCE_NAME_WS2);
212 
213         userTemplate.getMemberships().add(new MembershipTO.Builder("f779c0d4-633b-4be5-8f57-32eb478a3ca5").build());
214         task.getTemplates().put(AnyTypeKind.USER.name(), userTemplate);
215 
216         GroupTO groupTemplate = new GroupTO();
217         groupTemplate.getResources().add(RESOURCE_NAME_LDAP);
218         task.getTemplates().put(AnyTypeKind.GROUP.name(), groupTemplate);
219 
220         Response response = TASK_SERVICE.create(TaskType.PULL, task);
221         PullTaskTO actual = getObject(response.getLocation(), TaskService.class, PullTaskTO.class);
222         assertNotNull(actual);
223 
224         task = TASK_SERVICE.read(TaskType.PULL, actual.getKey(), true);
225         assertNotNull(task);
226         assertEquals(actual.getKey(), task.getKey());
227         assertEquals(actual.getJobDelegate(), task.getJobDelegate());
228         assertEquals(userTemplate, task.getTemplates().get(AnyTypeKind.USER.name()));
229         assertEquals(groupTemplate, task.getTemplates().get(AnyTypeKind.GROUP.name()));
230     }
231 
232     @Test
233     public void fromCSV() throws Exception {
234         assumeFalse(IS_EXT_SEARCH_ENABLED);
235 
236         removeTestUsers();
237 
238         // Attemp to reset CSV content
239         Properties props = new Properties();
240         InputStream propStream = null;
241         InputStream srcStream = null;
242         OutputStream dstStream = null;
243         try {
244             propStream = getClass().getResourceAsStream("/test.properties");
245             props.load(propStream);
246 
247             srcStream = new FileInputStream(props.getProperty("test.csv.src"));
248             dstStream = new FileOutputStream(props.getProperty("test.csv.dst"));
249 
250             IOUtils.copy(srcStream, dstStream);
251         } catch (IOException e) {
252             fail(e::getMessage);
253         } finally {
254             if (propStream != null) {
255                 propStream.close();
256             }
257             if (srcStream != null) {
258                 srcStream.close();
259             }
260             if (dstStream != null) {
261                 dstStream.close();
262             }
263         }
264 
265         // -----------------------------
266         // Create a new user ... it should be updated applying pull policy
267         // -----------------------------
268         UserCR inUserRC = new UserCR();
269         inUserRC.setRealm(SyncopeConstants.ROOT_REALM);
270         inUserRC.setPassword("password123");
271         String userName = "test9";
272         inUserRC.setUsername(userName);
273         inUserRC.getPlainAttrs().add(attr("firstname", "nome9"));
274         inUserRC.getPlainAttrs().add(attr("surname", "cognome"));
275         inUserRC.getPlainAttrs().add(attr("ctype", "a type"));
276         inUserRC.getPlainAttrs().add(attr("fullname", "nome cognome"));
277         inUserRC.getPlainAttrs().add(attr("userId", "puccini@syncope.apache.org"));
278         inUserRC.getPlainAttrs().add(attr("email", "puccini@syncope.apache.org"));
279         inUserRC.getAuxClasses().add("csv");
280 
281         UserTO inUserTO = createUser(inUserRC).getEntity();
282         assertNotNull(inUserTO);
283         assertFalse(inUserTO.getResources().contains(RESOURCE_NAME_CSV));
284 
285         // -----------------------------
286         try {
287             int usersPre = USER_SERVICE.search(new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
288                     page(1).size(1).build()).getTotalCount();
289             assertNotNull(usersPre);
290 
291             ExecTO exec = execProvisioningTask(TASK_SERVICE, TaskType.PULL, PULL_TASK_KEY, MAX_WAIT_SECONDS, false);
292             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(exec.getStatus()));
293 
294             LOG.debug("Execution of task {}:\n{}", PULL_TASK_KEY, exec);
295 
296             // check for pull results
297             int usersPost = USER_SERVICE.search(new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
298                     page(1).size(1).build()).getTotalCount();
299             assertNotNull(usersPost);
300             assertEquals(usersPre + 8, usersPost);
301 
302             // after execution of the pull task the user data should have been pulled from CSV
303             // and processed by user template
304             UserTO userTO = USER_SERVICE.read(inUserTO.getKey());
305             assertNotNull(userTO);
306             assertEquals(userName, userTO.getUsername());
307             assertEquals(IS_FLOWABLE_ENABLED
308                     ? "active" : "created", userTO.getStatus());
309             assertEquals("test9@syncope.apache.org", userTO.getPlainAttr("email").get().getValues().get(0));
310             assertEquals("test9@syncope.apache.org", userTO.getPlainAttr("userId").get().getValues().get(0));
311             assertTrue(Integer.valueOf(userTO.getPlainAttr("fullname").get().getValues().get(0)) <= 10);
312             assertTrue(userTO.getResources().contains(RESOURCE_NAME_TESTDB));
313             assertTrue(userTO.getResources().contains(RESOURCE_NAME_WS2));
314 
315             // Matching --> Update (no link)
316             assertFalse(userTO.getResources().contains(RESOURCE_NAME_CSV));
317 
318             // check for user template
319             userTO = USER_SERVICE.read("test7");
320             assertNotNull(userTO);
321             assertEquals("TYPE_OTHER", userTO.getPlainAttr("ctype").get().getValues().get(0));
322             assertEquals(3, userTO.getResources().size());
323             assertTrue(userTO.getResources().contains(RESOURCE_NAME_TESTDB));
324             assertTrue(userTO.getResources().contains(RESOURCE_NAME_WS2));
325             assertEquals(1, userTO.getMemberships().size());
326             assertEquals("f779c0d4-633b-4be5-8f57-32eb478a3ca5", userTO.getMemberships().get(0).getGroupKey());
327 
328             // Unmatching --> Assign (link) - SYNCOPE-658
329             assertTrue(userTO.getResources().contains(RESOURCE_NAME_CSV));
330             assertEquals(1, userTO.getDerAttrs().stream().
331                     filter(attrTO -> "csvuserid".equals(attrTO.getSchema())).count());
332 
333             userTO = USER_SERVICE.read("test8");
334             assertNotNull(userTO);
335             assertEquals("TYPE_8", userTO.getPlainAttr("ctype").get().getValues().get(0));
336 
337             // Check for ignored user - SYNCOPE-663
338             try {
339                 USER_SERVICE.read("test2");
340                 fail("This should not happen");
341             } catch (SyncopeClientException e) {
342                 assertEquals(Response.Status.NOT_FOUND, e.getType().getResponseStatus());
343             }
344 
345             // Check for issue 215:
346             // * expected disabled user test1
347             // * expected enabled user test3
348             userTO = USER_SERVICE.read("test1");
349             assertNotNull(userTO);
350             assertEquals("suspended", userTO.getStatus());
351 
352             userTO = USER_SERVICE.read("test3");
353             assertNotNull(userTO);
354             assertEquals("active", userTO.getStatus());
355 
356             Set<String> otherPullTaskKeys = Set.of(
357                     "feae4e57-15ca-40d9-b973-8b9015efca49",
358                     "55d5e74b-497e-4bc0-9156-73abef4b9adc");
359             execProvisioningTasks(TASK_SERVICE, TaskType.PULL, otherPullTaskKeys, MAX_WAIT_SECONDS, false);
360 
361             // Matching --> UNLINK
362             assertFalse(USER_SERVICE.read("test9").getResources().contains(RESOURCE_NAME_CSV));
363             assertFalse(USER_SERVICE.read("test7").getResources().contains(RESOURCE_NAME_CSV));
364         } finally {
365             removeTestUsers();
366         }
367     }
368 
369     @Test
370     public void dryRun() {
371         ExecTO execution = execProvisioningTask(TASK_SERVICE, TaskType.PULL, PULL_TASK_KEY, MAX_WAIT_SECONDS, true);
372         assertEquals("SUCCESS", execution.getStatus());
373     }
374 
375     @Test
376     public void reconcileFromDB() {
377         UserTO userTO = null;
378         JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
379         try {
380             ExecTO execution = execProvisioningTask(
381                     TASK_SERVICE, TaskType.PULL, "83f7e85d-9774-43fe-adba-ccd856312994", MAX_WAIT_SECONDS, false);
382             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
383 
384             userTO = USER_SERVICE.read("testuser1");
385             assertNotNull(userTO);
386             assertEquals("reconciled@syncope.apache.org", userTO.getPlainAttr("userId").get().getValues().get(0));
387             assertEquals("suspended", userTO.getStatus());
388 
389             // enable user on external resource
390             jdbcTemplate.execute("UPDATE TEST SET status=TRUE WHERE id='testuser1'");
391 
392             // re-execute the same PullTask: now user must be active
393             execution = execProvisioningTask(
394                     TASK_SERVICE, TaskType.PULL, "83f7e85d-9774-43fe-adba-ccd856312994", MAX_WAIT_SECONDS, false);
395             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
396 
397             userTO = USER_SERVICE.read("testuser1");
398             assertNotNull(userTO);
399             assertEquals("active", userTO.getStatus());
400         } finally {
401             jdbcTemplate.execute("UPDATE TEST SET status=FALSE WHERE id='testuser1'");
402             if (userTO != null) {
403                 USER_SERVICE.delete(userTO.getKey());
404             }
405         }
406     }
407 
408     @Test
409     public void reconcileFromLDAP() {
410         // First of all, clear any potential conflict with existing user / group
411         ldapCleanup();
412 
413         // 0. pull
414         ExecTO execution = execProvisioningTask(
415                 TASK_SERVICE, TaskType.PULL, "1e419ca4-ea81-4493-a14f-28b90113686d", MAX_WAIT_SECONDS, false);
416 
417         // 1. verify execution status
418         assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
419 
420         // SYNCOPE-898
421         PullTaskTO task = TASK_SERVICE.read(TaskType.PULL, "1e419ca4-ea81-4493-a14f-28b90113686d", false);
422         assertEquals(SyncopeConstants.ROOT_REALM, task.getDestinationRealm());
423 
424         if (IS_EXT_SEARCH_ENABLED) {
425             try {
426                 Thread.sleep(2000);
427             } catch (InterruptedException ex) {
428                 // ignore
429             }
430         }
431 
432         // 2. verify that pulled group is found
433         PagedResult<GroupTO> matchingGroups = GROUP_SERVICE.search(new AnyQuery.Builder().realm(
434                 SyncopeConstants.ROOT_REALM).
435                 fiql(SyncopeClient.getGroupSearchConditionBuilder().is("name").equalTo("testLDAPGroup").query()).
436                 build());
437         assertNotNull(matchingGroups);
438         assertEquals(1, matchingGroups.getResult().size());
439         assertEquals(SyncopeConstants.ROOT_REALM, matchingGroups.getResult().get(0).getRealm());
440 
441         // 3. verify that pulled user is found
442         PagedResult<UserTO> matchingUsers = USER_SERVICE.search(
443                 new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
444                         fiql(SyncopeClient.getUserSearchConditionBuilder().is("username").equalTo("pullFromLDAP").
445                                 query()).
446                         build());
447         assertNotNull(matchingUsers);
448         assertEquals(1, matchingUsers.getResult().size());
449         // SYNCOPE-898
450         assertEquals("/odd", matchingUsers.getResult().get(0).getRealm());
451 
452         // Check for SYNCOPE-436
453         assertEquals("pullFromLDAP",
454                 matchingUsers.getResult().get(0).getVirAttr("virtualReadOnly").get().getValues().get(0));
455         // Check for SYNCOPE-270
456         assertNotNull(matchingUsers.getResult().get(0).getPlainAttr("obscure"));
457         // Check for SYNCOPE-123
458         assertNotNull(matchingUsers.getResult().get(0).getPlainAttr("photo"));
459         // Check for SYNCOPE-1343
460         assertEquals("odd", matchingUsers.getResult().get(0).getPlainAttr("title").get().getValues().get(0));
461 
462         PagedResult<UserTO> matchByLastChangeContext = USER_SERVICE.search(
463                 new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
464                         fiql(SyncopeClient.getUserSearchConditionBuilder().is("lastChangeContext").
465                                 equalTo("*PullTask " + task.getKey() + "*").query()).
466                         build());
467         assertNotNull(matchByLastChangeContext);
468         assertNotEquals(0, matchByLastChangeContext.getTotalCount());
469 
470         GroupTO groupTO = matchingGroups.getResult().get(0);
471         assertNotNull(groupTO);
472         assertEquals("testLDAPGroup", groupTO.getName());
473         assertTrue(groupTO.getLastChangeContext().contains("PullTask " + task.getKey()));
474         assertEquals("true", groupTO.getPlainAttr("show").get().getValues().get(0));
475         assertEquals(matchingUsers.getResult().get(0).getKey(), groupTO.getUserOwner());
476         assertNull(groupTO.getGroupOwner());
477         // SYNCOPE-1343, set value title to null on LDAP
478         ConnObject userConnObject = RESOURCE_SERVICE.readConnObject(
479                 RESOURCE_NAME_LDAP, AnyTypeKind.USER.name(), matchingUsers.getResult().get(0).getKey());
480         assertNotNull(userConnObject);
481         assertEquals("odd", userConnObject.getAttr("title").get().getValues().get(0));
482         Attr userDn = userConnObject.getAttr(Name.NAME).get();
483         updateLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD,
484                 userDn.getValues().get(0), Collections.singletonMap("title", null));
485 
486         // SYNCOPE-317
487         execProvisioningTask(
488                 TASK_SERVICE, TaskType.PULL, "1e419ca4-ea81-4493-a14f-28b90113686d", MAX_WAIT_SECONDS, false);
489 
490         // 4. verify that LDAP group membership is pulled as Syncope membership
491         AtomicReference<Integer> numMembers = new AtomicReference<>();
492         await().atMost(MAX_WAIT_SECONDS, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> {
493             try {
494                 PagedResult<UserTO> members = USER_SERVICE.search(
495                         new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
496                                 fiql(SyncopeClient.getUserSearchConditionBuilder().inGroups(groupTO.getKey()).query()).
497                                 build());
498                 numMembers.set(members.getResult().size());
499                 return !members.getResult().isEmpty();
500             } catch (Exception e) {
501                 return false;
502             }
503         });
504         assertEquals(1, numMembers.get());
505 
506         // SYNCOPE-1343, verify that the title attribute has been reset
507         matchingUsers = USER_SERVICE.search(
508                 new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
509                         fiql(SyncopeClient.getUserSearchConditionBuilder().is("username").equalTo("pullFromLDAP").
510                                 query()).
511                         build());
512         assertNull(matchingUsers.getResult().get(0).getPlainAttr("title").orElse(null));
513 
514         // SYNCOPE-1356 remove group membership from LDAP, pull and check in Syncope
515         ConnObject groupConnObject = RESOURCE_SERVICE.readConnObject(
516                 RESOURCE_NAME_LDAP, AnyTypeKind.GROUP.name(), matchingGroups.getResult().get(0).getKey());
517         assertNotNull(groupConnObject);
518         Attr groupDn = groupConnObject.getAttr(Name.NAME).get();
519         updateLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD,
520                 groupDn.getValues().get(0), Map.of("uniquemember", "uid=admin,ou=system"));
521 
522         execProvisioningTask(
523                 TASK_SERVICE, TaskType.PULL, "1e419ca4-ea81-4493-a14f-28b90113686d", MAX_WAIT_SECONDS, false);
524 
525         await().atMost(MAX_WAIT_SECONDS, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> {
526             try {
527                 return USER_SERVICE.search(
528                         new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
529                                 fiql(SyncopeClient.getUserSearchConditionBuilder().inGroups(groupTO.getKey()).query()).
530                                 build()).getResult().isEmpty();
531             } catch (Exception e) {
532                 return false;
533             }
534         });
535     }
536 
537     @Test
538     public void reconcileFromScriptedSQL() throws IOException {
539         // 0. reset sync token and set MappingItemTransformer
540         ResourceTO resource = RESOURCE_SERVICE.read(RESOURCE_NAME_DBSCRIPTED);
541         ResourceTO originalResource = SerializationUtils.clone(resource);
542         Provision provision = resource.getProvision(PRINTER).get();
543         assertNotNull(provision);
544 
545         ImplementationTO transformer = null;
546         try {
547             transformer = IMPLEMENTATION_SERVICE.read(
548                     IdRepoImplementationType.ITEM_TRANSFORMER, "PrefixItemTransformer");
549         } catch (SyncopeClientException e) {
550             if (e.getType().getResponseStatus() == Response.Status.NOT_FOUND) {
551                 transformer = new ImplementationTO();
552                 transformer.setKey("PrefixItemTransformer");
553                 transformer.setEngine(ImplementationEngine.GROOVY);
554                 transformer.setType(IdRepoImplementationType.ITEM_TRANSFORMER);
555                 transformer.setBody(IOUtils.toString(
556                         getClass().getResourceAsStream("/PrefixItemTransformer.groovy"), StandardCharsets.UTF_8));
557                 Response response = IMPLEMENTATION_SERVICE.create(transformer);
558                 transformer = IMPLEMENTATION_SERVICE.read(
559                         transformer.getType(), response.getHeaderString(RESTHeaders.RESOURCE_KEY));
560                 assertNotNull(transformer.getKey());
561             }
562         }
563         assertNotNull(transformer);
564 
565         Item mappingItem = provision.getMapping().getItems().stream().
566                 filter(object -> "location".equals(object.getIntAttrName())).findFirst().get();
567         assertNotNull(mappingItem);
568         mappingItem.getTransformers().clear();
569         mappingItem.getTransformers().add(transformer.getKey());
570 
571         final String prefix = "PREFIX_";
572         try {
573             RESOURCE_SERVICE.update(resource);
574             RESOURCE_SERVICE.removeSyncToken(resource.getKey(), provision.getAnyType());
575 
576             // insert a deleted record in the external resource (SYNCOPE-923), which will be returned
577             // as sync event prior to the CREATE_OR_UPDATE events generated by the actions below (before pull)
578             JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
579             jdbcTemplate.update(
580                     "INSERT INTO TESTPRINTER (id, printername, location, deleted, lastmodification) VALUES (?,?,?,?,?)",
581                     UUID.randomUUID().toString(), "Mysterious Printer", "Nowhere", true, new Date());
582 
583             // 1. create printer on external resource
584             AnyObjectCR anyObjectCR = AnyObjectITCase.getSample("pull");
585             AnyObjectTO anyObjectTO = createAnyObject(anyObjectCR).getEntity();
586             assertNotNull(anyObjectTO);
587             String originalLocation = anyObjectTO.getPlainAttr("location").get().getValues().get(0);
588             assertFalse(originalLocation.startsWith(prefix));
589 
590             // 2. verify that PrefixMappingItemTransformer was applied during propagation
591             // (location starts with given prefix on external resource)
592             ConnObject connObjectTO = RESOURCE_SERVICE.readConnObject(
593                     RESOURCE_NAME_DBSCRIPTED, anyObjectTO.getType(), anyObjectTO.getKey());
594             assertFalse(anyObjectTO.getPlainAttr("location").get().getValues().get(0).startsWith(prefix));
595             assertTrue(connObjectTO.getAttr("LOCATION").get().getValues().get(0).startsWith(prefix));
596 
597             // 3. unlink any existing printer and delete from Syncope (printer is now only on external resource)
598             if (IS_EXT_SEARCH_ENABLED) {
599                 try {
600                     Thread.sleep(2000);
601                 } catch (InterruptedException ex) {
602                     // ignore
603                 }
604             }
605 
606             PagedResult<AnyObjectTO> matchingPrinters = ANY_OBJECT_SERVICE.search(
607                     new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
608                             fiql(SyncopeClient.getAnyObjectSearchConditionBuilder(PRINTER).
609                                     is("location").equalTo("pull*").query()).build());
610             assertTrue(matchingPrinters.getSize() > 0);
611             for (AnyObjectTO printer : matchingPrinters.getResult()) {
612                 ANY_OBJECT_SERVICE.deassociate(new ResourceDR.Builder().key(printer.getKey()).
613                         action(ResourceDeassociationAction.UNLINK).resource(RESOURCE_NAME_DBSCRIPTED).build());
614                 ANY_OBJECT_SERVICE.delete(printer.getKey());
615             }
616 
617             // ensure that the pull task does not have the DELETE capability (SYNCOPE-923)
618             PullTaskTO pullTask = TASK_SERVICE.read(TaskType.PULL, "30cfd653-257b-495f-8665-281281dbcb3d", false);
619             assertNotNull(pullTask);
620             assertFalse(pullTask.isPerformDelete());
621 
622             // 4. pull
623             execProvisioningTask(TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
624 
625             if (IS_EXT_SEARCH_ENABLED) {
626                 try {
627                     Thread.sleep(2000);
628                 } catch (InterruptedException ex) {
629                     // ignore
630                 }
631             }
632 
633             // 5. verify that printer was re-created in Syncope (implies that location does not start with given prefix,
634             // hence PrefixItemTransformer was applied during pull)
635             if (IS_EXT_SEARCH_ENABLED) {
636                 try {
637                     Thread.sleep(2000);
638                 } catch (InterruptedException ex) {
639                     // ignore
640                 }
641             }
642 
643             matchingPrinters = ANY_OBJECT_SERVICE.search(new AnyQuery.Builder().realm(SyncopeConstants.ROOT_REALM).
644                     fiql(SyncopeClient.getAnyObjectSearchConditionBuilder(PRINTER).
645                             is("location").equalTo("pull*").query()).build());
646             assertTrue(matchingPrinters.getSize() > 0);
647 
648             // 6. verify that synctoken was updated
649             assertNotNull(RESOURCE_SERVICE.read(RESOURCE_NAME_DBSCRIPTED).
650                     getProvision(anyObjectTO.getType()).get().getSyncToken());
651         } finally {
652             RESOURCE_SERVICE.update(originalResource);
653         }
654     }
655 
656     @Test
657     public void filteredReconciliation() throws IOException {
658         String user1OnTestPull = UUID.randomUUID().toString();
659         String user2OnTestPull = UUID.randomUUID().toString();
660 
661         JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
662         PullTaskTO task = null;
663         UserTO userTO = null;
664         try {
665             // 1. create 2 users on testpull
666             jdbcTemplate.execute("INSERT INTO testpull VALUES ("
667                     + '\'' + user1OnTestPull + "', 'user1', 'Doe', false, 'mail1@apache.org', NULL)");
668             jdbcTemplate.execute("INSERT INTO testpull VALUES ("
669                     + '\'' + user2OnTestPull + "', 'user2', 'Rossi', false, 'mail2@apache.org', NULL)");
670 
671             // 2. create new pull task for test-db, with reconciliation filter (surname 'Rossi') 
672             ImplementationTO reconFilterBuilder = new ImplementationTO();
673             reconFilterBuilder.setKey("TestReconFilterBuilder");
674             reconFilterBuilder.setEngine(ImplementationEngine.GROOVY);
675             reconFilterBuilder.setType(IdMImplementationType.RECON_FILTER_BUILDER);
676             reconFilterBuilder.setBody(IOUtils.toString(
677                     getClass().getResourceAsStream("/TestReconFilterBuilder.groovy"), StandardCharsets.UTF_8));
678             Response response = IMPLEMENTATION_SERVICE.create(reconFilterBuilder);
679             reconFilterBuilder = IMPLEMENTATION_SERVICE.read(
680                     reconFilterBuilder.getType(), response.getHeaderString(RESTHeaders.RESOURCE_KEY));
681             assertNotNull(reconFilterBuilder);
682 
683             task = TASK_SERVICE.read(TaskType.PULL, "7c2242f4-14af-4ab5-af31-cdae23783655", true);
684             task.setName(getUUIDString());
685             task.setPullMode(PullMode.FILTERED_RECONCILIATION);
686             task.setReconFilterBuilder(reconFilterBuilder.getKey());
687             response = TASK_SERVICE.create(TaskType.PULL, task);
688             task = getObject(response.getLocation(), TaskService.class, PullTaskTO.class);
689             assertNotNull(task);
690             assertEquals(reconFilterBuilder.getKey(), task.getReconFilterBuilder());
691 
692             // 3. exec task
693             ExecTO execution = execProvisioningTask(
694                     TASK_SERVICE, TaskType.PULL, task.getKey(), MAX_WAIT_SECONDS, false);
695             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
696 
697             // 4. verify that only enabled user was pulled
698             userTO = USER_SERVICE.read("user2");
699             assertNotNull(userTO);
700 
701             try {
702                 USER_SERVICE.read("user1");
703                 fail("This should not happen");
704             } catch (SyncopeClientException e) {
705                 assertEquals(ClientExceptionType.NotFound, e.getType());
706             }
707         } finally {
708             jdbcTemplate.execute("DELETE FROM testpull WHERE id = '" + user1OnTestPull + '\'');
709             jdbcTemplate.execute("DELETE FROM testpull WHERE id = '" + user2OnTestPull + '\'');
710             if (task != null && !"7c2242f4-14af-4ab5-af31-cdae23783655".equals(task.getKey())) {
711                 TASK_SERVICE.delete(TaskType.PULL, task.getKey());
712             }
713             if (userTO != null) {
714                 USER_SERVICE.delete(userTO.getKey());
715             }
716         }
717     }
718 
719     @Test
720     public void syncTokenWithErrors() {
721         ResourceTO origResource = RESOURCE_SERVICE.read(RESOURCE_NAME_DBPULL);
722         ConnInstanceTO origConnector = CONNECTOR_SERVICE.read(origResource.getConnector(), null);
723 
724         ResourceTO resForTest = SerializationUtils.clone(origResource);
725         resForTest.setKey("syncTokenWithErrors");
726         resForTest.setConnector(null);
727         ConnInstanceTO connForTest = SerializationUtils.clone(origConnector);
728         connForTest.setKey(null);
729         connForTest.setDisplayName("For syncTokenWithErrors");
730 
731         JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
732         try {
733             connForTest.getCapabilities().add(ConnectorCapability.SYNC);
734 
735             ConnConfProperty changeLogColumn = connForTest.getConf("changeLogColumn").get();
736             assertNotNull(changeLogColumn);
737             assertTrue(changeLogColumn.getValues().isEmpty());
738             changeLogColumn.getValues().add("lastModification");
739 
740             Response response = CONNECTOR_SERVICE.create(connForTest);
741             if (response.getStatusInfo().getStatusCode() != Response.Status.CREATED.getStatusCode()) {
742                 throw (RuntimeException) CLIENT_FACTORY.getExceptionMapper().fromResponse(response);
743             }
744             connForTest = getObject(response.getLocation(), ConnectorService.class, ConnInstanceTO.class);
745             assertNotNull(connForTest);
746 
747             resForTest.setConnector(connForTest.getKey());
748             resForTest = createResource(resForTest);
749             assertNotNull(resForTest);
750 
751             PullTaskTO pullTask = new PullTaskTO();
752             pullTask.setActive(true);
753             pullTask.setName("For syncTokenWithErrors");
754             pullTask.setResource(resForTest.getKey());
755             pullTask.setDestinationRealm(SyncopeConstants.ROOT_REALM);
756             pullTask.setPullMode(PullMode.INCREMENTAL);
757             pullTask.setPerformCreate(true);
758             pullTask.setPerformUpdate(true);
759             pullTask.setPerformDelete(true);
760 
761             response = TASK_SERVICE.create(TaskType.PULL, pullTask);
762             if (response.getStatusInfo().getStatusCode() != Response.Status.CREATED.getStatusCode()) {
763                 throw (RuntimeException) CLIENT_FACTORY.getExceptionMapper().fromResponse(response);
764             }
765             pullTask = getObject(response.getLocation(), TaskService.class, PullTaskTO.class);
766             assertNotNull(pullTask);
767 
768             jdbcTemplate.execute("DELETE FROM testpull");
769             jdbcTemplate.execute("INSERT INTO testpull VALUES "
770                     + "(1040, 'syncTokenWithErrors1', 'Surname1', "
771                     + "false, 'syncTokenWithErrors1@syncope.apache.org', '2014-05-23 13:53:24.293')");
772             jdbcTemplate.execute("INSERT INTO testpull VALUES "
773                     + "(1041, 'syncTokenWithErrors2', 'Surname2', "
774                     + "false, 'syncTokenWithErrors1@syncope.apache.org', '2015-05-23 13:53:24.293')");
775 
776             ExecTO exec = execProvisioningTask(TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
777             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(exec.getStatus()));
778 
779             resForTest = RESOURCE_SERVICE.read(resForTest.getKey());
780             assertTrue(resForTest.getProvision(AnyTypeKind.USER.name()).get().getSyncToken().contains("2014-05-23"));
781 
782             jdbcTemplate.execute("UPDATE testpull "
783                     + "SET email='syncTokenWithErrors2@syncope.apache.org', lastModification='2016-05-23 13:53:24.293' "
784                     + "WHERE ID=1041");
785 
786             exec = execProvisioningTask(TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
787             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(exec.getStatus()));
788 
789             resForTest = RESOURCE_SERVICE.read(resForTest.getKey());
790             assertTrue(resForTest.getProvision(AnyTypeKind.USER.name()).get().getSyncToken().contains("2016-05-23"));
791         } finally {
792             if (resForTest.getConnector() != null) {
793                 RESOURCE_SERVICE.delete(resForTest.getKey());
794                 CONNECTOR_SERVICE.delete(connForTest.getKey());
795             }
796 
797             jdbcTemplate.execute("DELETE FROM testpull WHERE ID=1040");
798             jdbcTemplate.execute("DELETE FROM testpull WHERE ID=1041");
799         }
800     }
801 
802     @Test
803     public void remediation() {
804         // First of all, clear any potential conflict with existing user / group
805         ldapCleanup();
806 
807         // 1. create ldap cloned resource, where 'userId' (mandatory on Syncope) is removed from mapping
808         ResourceTO ldap = RESOURCE_SERVICE.read(RESOURCE_NAME_LDAP);
809         ldap.setKey("ldapForRemediation");
810 
811         Provision provision = ldap.getProvision(AnyTypeKind.USER.name()).get();
812         provision.getMapping().getItems().removeIf(item -> "userId".equals(item.getIntAttrName()));
813         provision.getMapping().getItems().removeIf(item -> "mail".equals(item.getIntAttrName()));
814         provision.getVirSchemas().clear();
815 
816         ldap.getProvisions().clear();
817         ldap.getProvisions().add(provision);
818 
819         ldap = createResource(ldap);
820 
821         // 2. create PullTask with remediation enabled, for the new resource
822         PullTaskTO pullTask = (PullTaskTO) TASK_SERVICE.search(new TaskQuery.Builder(TaskType.PULL).
823                 resource(RESOURCE_NAME_LDAP).build()).getResult().get(0);
824         assertNotNull(pullTask);
825         pullTask.setName(getUUIDString());
826         pullTask.setResource(ldap.getKey());
827         pullTask.setRemediation(true);
828         pullTask.getActions().clear();
829 
830         Response response = TASK_SERVICE.create(TaskType.PULL, pullTask);
831         if (response.getStatusInfo().getStatusCode() != Response.Status.CREATED.getStatusCode()) {
832             throw (RuntimeException) CLIENT_FACTORY.getExceptionMapper().fromResponse(response);
833         }
834         pullTask = getObject(response.getLocation(), TaskService.class, PullTaskTO.class);
835         assertNotNull(pullTask);
836 
837         try {
838             // 3. execute the pull task and verify that:
839             ExecTO execution = execProvisioningTask(
840                     TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
841             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
842 
843             // 3a. user was not pulled
844             try {
845                 USER_SERVICE.read("pullFromLDAP");
846                 fail("This should never happen");
847             } catch (SyncopeClientException e) {
848                 assertEquals(ClientExceptionType.NotFound, e.getType());
849             }
850 
851             // 3b. remediation was created
852             Optional<RemediationTO> remediation = REMEDIATION_SERVICE.list(
853                     new RemediationQuery.Builder().page(1).size(1000).build()).getResult().stream().
854                     filter(r -> "uid=pullFromLDAP,ou=People,o=isp".equalsIgnoreCase(r.getRemoteName())).
855                     findFirst();
856             assertTrue(remediation.isPresent());
857             assertEquals(AnyTypeKind.USER.name(), remediation.get().getAnyType());
858             assertEquals(ResourceOperation.CREATE, remediation.get().getOperation());
859             assertNotNull(remediation.get().getAnyCRPayload());
860             assertNull(remediation.get().getAnyURPayload());
861             assertNull(remediation.get().getKeyPayload());
862             assertTrue(remediation.get().getError().contains("RequiredValuesMissing [userId]"));
863 
864             // 4. remedy by copying the email value to userId
865             AnyCR userCR = remediation.get().getAnyCRPayload();
866             userCR.getResources().clear();
867 
868             String email = userCR.getPlainAttr("email").get().getValues().get(0);
869             userCR.getPlainAttrs().add(new Attr.Builder("userId").value(email).build());
870 
871             REMEDIATION_SERVICE.remedy(remediation.get().getKey(), userCR);
872 
873             // 5. user is now found
874             UserTO user = USER_SERVICE.read("pullFromLDAP");
875             assertNotNull(user);
876             assertEquals(email, user.getPlainAttr("userId").get().getValues().get(0));
877 
878             // 6. remediation was removed
879             try {
880                 REMEDIATION_SERVICE.read(remediation.get().getKey());
881                 fail("This should never happen");
882             } catch (SyncopeClientException e) {
883                 assertEquals(ClientExceptionType.NotFound, e.getType());
884             }
885         } finally {
886             RESOURCE_SERVICE.delete(ldap.getKey());
887         }
888     }
889 
890     @Test
891     public void remediationSinglePull() throws IOException {
892         // First of all, clear any potential conflict with existing user / group
893         ldapCleanup();
894 
895         ResourceTO ldap = RESOURCE_SERVICE.read(RESOURCE_NAME_LDAP);
896         ldap.setKey("ldapForRemediationSinglePull");
897 
898         Provision provision = ldap.getProvision(AnyTypeKind.USER.name()).get();
899         provision.getMapping().getItems().removeIf(item -> "userId".equals(item.getIntAttrName()));
900         provision.getMapping().getItems().removeIf(item -> "email".equals(item.getIntAttrName()));
901         provision.getVirSchemas().clear();
902 
903         ldap.getProvisions().clear();
904         ldap.getProvisions().add(provision);
905 
906         ldap = createResource(ldap);
907 
908         try {
909             // 2. pull an user
910             PullTaskTO pullTask = new PullTaskTO();
911             pullTask.setResource(ldap.getKey());
912             pullTask.setDestinationRealm(SyncopeConstants.ROOT_REALM);
913             pullTask.setRemediation(true);
914             pullTask.setPerformCreate(true);
915             pullTask.setPerformUpdate(true);
916             pullTask.setUnmatchingRule(UnmatchingRule.ASSIGN);
917             pullTask.setMatchingRule(MatchingRule.UPDATE);
918 
919             try {
920                 RECONCILIATION_SERVICE.pull(new ReconQuery.Builder(AnyTypeKind.USER.name(), ldap.getKey()).
921                         fiql("uid==pullFromLDAP").build(), pullTask);
922                 fail("Should not arrive here");
923             } catch (SyncopeClientException sce) {
924                 assertEquals(ClientExceptionType.Reconciliation, sce.getType());
925             }
926             Optional<RemediationTO> remediation = REMEDIATION_SERVICE.list(
927                     new RemediationQuery.Builder().after(OffsetDateTime.now().minusSeconds(30)).
928                             page(1).size(1000).build()).getResult().stream().
929                     filter(r -> "uid=pullFromLDAP,ou=People,o=isp".equalsIgnoreCase(r.getRemoteName())).
930                     findFirst();
931             assertTrue(remediation.isPresent());
932             assertEquals(AnyTypeKind.USER.name(), remediation.get().getAnyType());
933             assertEquals(ResourceOperation.CREATE, remediation.get().getOperation());
934             assertNotNull(remediation.get().getAnyCRPayload());
935             assertNull(remediation.get().getAnyURPayload());
936             assertNull(remediation.get().getKeyPayload());
937             assertTrue(remediation.get().getError().contains(
938                     "SyncopeClientCompositeException: {[RequiredValuesMissing [userId]]}"));
939         } finally {
940             RESOURCE_SERVICE.delete(ldap.getKey());
941             cleanUpRemediations();
942         }
943     }
944 
945     @Test
946     public void concurrentPull() throws NamingException, InterruptedException {
947         int usersBefore = USER_SERVICE.search(new AnyQuery.Builder().fiql(
948                 SyncopeClient.getUserSearchConditionBuilder().is("username").equalTo("pullFromLDAP_*").query()).
949                 page(1).size(0).build()).getTotalCount();
950 
951         // 0. first cleanup then create 20 users on LDAP
952         ldapCleanup();
953 
954         ExecutorService tp = Executors.newFixedThreadPool(10);
955         for (int i = 0; i < 20; i++) {
956             String idx = StringUtils.leftPad(String.valueOf(i), 2, "0");
957             tp.submit(() -> {
958                 try {
959                     createLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD, prepareLdapAttributes(
960                             "pullFromLDAP_" + idx,
961                             "pullFromLDAP_" + idx + "@syncope.apache.org",
962                             "Active",
963                             "pullFromLDAP_" + idx,
964                             "Surname",
965                             "5BAA61E4C9B93F3F0682250B6CF8331B7EE68FD8",
966                             "odd",
967                             "password"));
968                 } catch (NamingException e) {
969                     LOG.error("While creating LDAP {}-th user", idx, e);
970                 }
971             });
972         }
973         tp.shutdown();
974         tp.awaitTermination(MAX_WAIT_SECONDS, TimeUnit.SECONDS);
975 
976         // 1. create new concurrent pull task
977         PullTaskTO pullTask = TASK_SERVICE.read(TaskType.PULL, "1e419ca4-ea81-4493-a14f-28b90113686d", false);
978         assertNull(pullTask.getConcurrentSettings());
979         pullTask.setKey(null);
980         pullTask.setName("LDAP Concurrent Pull Task");
981         pullTask.setDescription("LDAP Concurrent Pull Task");
982 
983         ThreadPoolSettings tps = new ThreadPoolSettings();
984         tps.setCorePoolSize(1);
985         tps.setMaxPoolSize(2);
986         tps.setQueueCapacity(40);
987         pullTask.setConcurrentSettings(tps);
988 
989         Response response = TASK_SERVICE.create(TaskType.PULL, pullTask);
990         String pullTaskKey = response.getHeaderString(RESTHeaders.RESOURCE_KEY);
991 
992         PagedResult<UserTO> result = null;
993         try {
994             // 2. run concurrent pull task
995             ExecTO execution = execProvisioningTask(
996                     TASK_SERVICE, TaskType.PULL, pullTaskKey, 2 * MAX_WAIT_SECONDS, false);
997 
998             // 3. verify execution status
999             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1000 
1001             // 4. verify that the given number of users was effectively pulled
1002             result = USER_SERVICE.search(new AnyQuery.Builder().fiql(
1003                     SyncopeClient.getUserSearchConditionBuilder().is("username").equalTo("pullFromLDAP_*").query()).
1004                     page(1).size(100).build());
1005             assertTrue(result.getTotalCount() > usersBefore);
1006         } finally {
1007             if (result != null) {
1008                 BatchRequest batchRequest = ADMIN_CLIENT.batch();
1009                 UserService batchUserService = batchRequest.getService(UserService.class);
1010                 result.getResult().stream().map(UserTO::getKey).forEach(batchUserService::delete);
1011                 batchRequest.commit();
1012             }
1013         }
1014     }
1015 
1016     @Test
1017     public void issueSYNCOPE68() {
1018         //-----------------------------
1019         // Create a new user ... it should be updated applying pull policy
1020         //-----------------------------
1021         UserCR userCR = new UserCR();
1022         userCR.setRealm(SyncopeConstants.ROOT_REALM);
1023         userCR.setPassword("password123");
1024         userCR.setUsername("testuser2");
1025 
1026         userCR.getPlainAttrs().add(attr("firstname", "testuser2"));
1027         userCR.getPlainAttrs().add(attr("surname", "testuser2"));
1028         userCR.getPlainAttrs().add(attr("ctype", "a type"));
1029         userCR.getPlainAttrs().add(attr("fullname", "a type"));
1030         userCR.getPlainAttrs().add(attr("userId", "testuser2@syncope.apache.org"));
1031         userCR.getPlainAttrs().add(attr("email", "testuser2@syncope.apache.org"));
1032 
1033         userCR.getResources().add(RESOURCE_NAME_NOPROPAGATION2);
1034         userCR.getResources().add(RESOURCE_NAME_NOPROPAGATION4);
1035 
1036         userCR.getMemberships().add(new MembershipTO.Builder("bf825fe1-7320-4a54-bd64-143b5c18ab97").build());
1037 
1038         UserTO userTO = createUser(userCR).getEntity();
1039         assertNotNull(userTO);
1040         assertEquals("testuser2", userTO.getUsername());
1041         assertEquals(1, userTO.getMemberships().size());
1042         assertEquals(3, userTO.getResources().size());
1043         //-----------------------------
1044 
1045         try {
1046             //-----------------------------
1047             //  add user template
1048             //-----------------------------
1049             UserTO template = new UserTO();
1050 
1051             template.getMemberships().add(new MembershipTO.Builder("b8d38784-57e7-4595-859a-076222644b55").build());
1052 
1053             template.getResources().add(RESOURCE_NAME_NOPROPAGATION4);
1054             //-----------------------------
1055 
1056             // Update pull task
1057             PullTaskTO task = TASK_SERVICE.read(TaskType.PULL, "81d88f73-d474-4450-9031-605daa4e313f", true);
1058             assertNotNull(task);
1059 
1060             task.getTemplates().put(AnyTypeKind.USER.name(), template);
1061 
1062             TASK_SERVICE.update(TaskType.PULL, task);
1063             PullTaskTO actual = TASK_SERVICE.read(TaskType.PULL, task.getKey(), true);
1064             assertNotNull(actual);
1065             assertEquals(task.getKey(), actual.getKey());
1066             assertFalse(actual.getTemplates().get(AnyTypeKind.USER.name()).getResources().isEmpty());
1067             assertFalse(((UserTO) actual.getTemplates().get(AnyTypeKind.USER.name())).getMemberships().isEmpty());
1068 
1069             ExecTO execution = execProvisioningTask(
1070                     TASK_SERVICE, TaskType.PULL, actual.getKey(), MAX_WAIT_SECONDS, false);
1071             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1072 
1073             userTO = USER_SERVICE.read("testuser2");
1074             assertNotNull(userTO);
1075             assertEquals("testuser2@syncope.apache.org", userTO.getPlainAttr("userId").get().getValues().get(0));
1076             assertEquals(2, userTO.getMemberships().size());
1077             assertEquals(4, userTO.getResources().size());
1078         } finally {
1079             UserTO dUserTO = deleteUser(userTO.getKey()).getEntity();
1080             assertNotNull(dUserTO);
1081         }
1082     }
1083 
1084     @Test
1085     public void issueSYNCOPE230() {
1086         JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
1087 
1088         String id = "a54b3794-b231-47be-b24a-11e1a42949f6";
1089 
1090         // 1. populate the external table
1091         jdbcTemplate.execute("INSERT INTO testpull VALUES"
1092                 + "('" + id + "', 'issuesyncope230', 'Surname230', false, 'syncope230@syncope.apache.org', NULL)");
1093 
1094         // 2. execute PullTask for resource-db-pull (table TESTPULL on external H2)
1095         execProvisioningTask(
1096                 TASK_SERVICE, TaskType.PULL, "7c2242f4-14af-4ab5-af31-cdae23783655", MAX_WAIT_SECONDS, false);
1097 
1098         // 3. read e-mail address for user created by the PullTask first execution
1099         UserTO userTO = USER_SERVICE.read("issuesyncope230");
1100         assertNotNull(userTO);
1101         String email = userTO.getPlainAttr("email").get().getValues().iterator().next();
1102         assertNotNull(email);
1103 
1104         // 4. update TESTPULL on external H2 by changing e-mail address
1105         jdbcTemplate.execute("UPDATE TESTPULL SET email='updatedSYNCOPE230@syncope.apache.org' WHERE id='" + id + '\'');
1106 
1107         // 5. re-execute the PullTask
1108         execProvisioningTask(
1109                 TASK_SERVICE, TaskType.PULL, "7c2242f4-14af-4ab5-af31-cdae23783655", MAX_WAIT_SECONDS, false);
1110 
1111         // 6. verify that the e-mail was updated
1112         userTO = USER_SERVICE.read("issuesyncope230");
1113         assertNotNull(userTO);
1114         email = userTO.getPlainAttr("email").get().getValues().iterator().next();
1115         assertNotNull(email);
1116         assertEquals("updatedSYNCOPE230@syncope.apache.org", email);
1117     }
1118 
1119     @Test
1120     public void issueSYNCOPE258() throws IOException {
1121         assumeFalse(IS_EXT_SEARCH_ENABLED);
1122 
1123         // -----------------------------
1124         // Add a custom correlation rule
1125         // -----------------------------
1126         ImplementationTO corrRule = null;
1127         try {
1128             corrRule = IMPLEMENTATION_SERVICE.read(IdMImplementationType.PULL_CORRELATION_RULE, "TestPullRule");
1129         } catch (SyncopeClientException e) {
1130             if (e.getType().getResponseStatus() == Response.Status.NOT_FOUND) {
1131                 corrRule = new ImplementationTO();
1132                 corrRule.setKey("TestPullRule");
1133                 corrRule.setEngine(ImplementationEngine.GROOVY);
1134                 corrRule.setType(IdMImplementationType.PULL_CORRELATION_RULE);
1135                 corrRule.setBody(IOUtils.toString(
1136                         getClass().getResourceAsStream("/TestPullRule.groovy"), StandardCharsets.UTF_8));
1137                 Response response = IMPLEMENTATION_SERVICE.create(corrRule);
1138                 corrRule = IMPLEMENTATION_SERVICE.read(
1139                         corrRule.getType(), response.getHeaderString(RESTHeaders.RESOURCE_KEY));
1140                 assertNotNull(corrRule);
1141             }
1142         }
1143         assertNotNull(corrRule);
1144 
1145         PullPolicyTO policyTO = POLICY_SERVICE.read(PolicyType.PULL, "9454b0d7-2610-400a-be82-fc23cf553dd6");
1146         policyTO.getCorrelationRules().put(AnyTypeKind.USER.name(), corrRule.getKey());
1147         POLICY_SERVICE.update(PolicyType.PULL, policyTO);
1148         // -----------------------------
1149 
1150         PullTaskTO task = new PullTaskTO();
1151         task.setDestinationRealm(SyncopeConstants.ROOT_REALM);
1152         task.setName(getUUIDString());
1153         task.setActive(true);
1154         task.setResource(RESOURCE_NAME_WS2);
1155         task.setPullMode(PullMode.FULL_RECONCILIATION);
1156         task.setPerformCreate(true);
1157         task.setPerformDelete(true);
1158         task.setPerformUpdate(true);
1159 
1160         Response response = TASK_SERVICE.create(TaskType.PULL, task);
1161         task = getObject(response.getLocation(), TaskService.class, PullTaskTO.class);
1162 
1163         UserCR userCR = UserITCase.getUniqueSample("s258_1@apache.org");
1164         userCR.getResources().clear();
1165         userCR.getResources().add(RESOURCE_NAME_WS2);
1166 
1167         createUser(userCR);
1168 
1169         userCR = UserITCase.getUniqueSample("s258_2@apache.org");
1170         userCR.getResources().clear();
1171         userCR.getResources().add(RESOURCE_NAME_WS2);
1172 
1173         UserTO userTO = createUser(userCR).getEntity();
1174 
1175         // change email in order to unmatch the second user
1176         UserUR userUR = new UserUR();
1177         userUR.setKey(userTO.getKey());
1178         userUR.getPlainAttrs().add(attrAddReplacePatch("email", "s258@apache.org"));
1179 
1180         USER_SERVICE.update(userUR);
1181 
1182         execProvisioningTask(TASK_SERVICE, TaskType.PULL, task.getKey(), MAX_WAIT_SECONDS, false);
1183 
1184         PullTaskTO executed = TASK_SERVICE.read(TaskType.PULL, task.getKey(), true);
1185         assertEquals(1, executed.getExecutions().size());
1186 
1187         // asser for just one match
1188         assertTrue(executed.getExecutions().get(0).getMessage().contains("[updated/failures]: 1/0"));
1189     }
1190 
1191     @Test
1192     public void issueSYNCOPE272() {
1193         removeTestUsers();
1194 
1195         // create user with testdb resource
1196         UserCR userCR = UserITCase.getUniqueSample("syncope272@syncope.apache.org");
1197         userCR.getResources().add(RESOURCE_NAME_TESTDB);
1198 
1199         ProvisioningResult<UserTO> result = createUser(userCR);
1200         UserTO userTO = result.getEntity();
1201         try {
1202             assertNotNull(userTO);
1203             assertEquals(1, result.getPropagationStatuses().size());
1204             assertEquals(ExecStatus.SUCCESS, result.getPropagationStatuses().get(0).getStatus());
1205 
1206             ExecTO taskExecTO = execProvisioningTask(
1207                     TASK_SERVICE, TaskType.PULL, "986867e2-993b-430e-8feb-aa9abb4c1dcd", MAX_WAIT_SECONDS, false);
1208 
1209             assertNotNull(taskExecTO.getStatus());
1210             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(taskExecTO.getStatus()));
1211 
1212             userTO = USER_SERVICE.read(userTO.getKey());
1213             assertNotNull(userTO);
1214             assertNotNull(userTO.getPlainAttr("firstname").get().getValues().get(0));
1215         } finally {
1216             removeTestUsers();
1217         }
1218     }
1219 
1220     @Test
1221     public void issueSYNCOPE307() {
1222         assumeFalse(IS_EXT_SEARCH_ENABLED);
1223 
1224         UserCR userCR = UserITCase.getUniqueSample("s307@apache.org");
1225         userCR.setUsername("test0");
1226         userCR.getPlainAttrs().removeIf(attr -> "firstname".equals(attr.getSchema()));
1227         userCR.getPlainAttrs().add(attr("firstname", "nome0"));
1228         userCR.getAuxClasses().add("csv");
1229 
1230         userCR.getResources().clear();
1231         userCR.getResources().add(RESOURCE_NAME_WS2);
1232 
1233         UserTO userTO = createUser(userCR).getEntity();
1234         assertNotNull(userTO);
1235 
1236         userTO = USER_SERVICE.read(userTO.getKey());
1237         assertTrue(userTO.getVirAttrs().isEmpty());
1238 
1239         // Update pull task
1240         PullTaskTO task = TASK_SERVICE.read(TaskType.PULL, "38abbf9e-a1a3-40a1-a15f-7d0ac02f47f1", true);
1241         assertNotNull(task);
1242 
1243         UserTO template = new UserTO();
1244         template.setPassword("'password123'");
1245         template.getResources().add(RESOURCE_NAME_DBVIRATTR);
1246         template.getVirAttrs().add(attr("virtualdata", "'virtualvalue'"));
1247 
1248         task.getTemplates().put(AnyTypeKind.USER.name(), template);
1249 
1250         TASK_SERVICE.update(TaskType.PULL, task);
1251 
1252         // exec task: one user from CSV will match the user created above and template will be applied
1253         ExecTO exec = execProvisioningTask(TASK_SERVICE, TaskType.PULL, task.getKey(), MAX_WAIT_SECONDS, false);
1254 
1255         // check that template was successfully applied
1256         // 1. propagation to db
1257         assertEquals(ExecStatus.SUCCESS.name(), exec.getStatus());
1258 
1259         JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
1260         String value = queryForObject(jdbcTemplate,
1261                 MAX_WAIT_SECONDS, "SELECT USERNAME FROM testpull WHERE ID=?", String.class, userTO.getKey());
1262         assertEquals("virtualvalue", value);
1263 
1264         // 2. virtual attribute
1265         userTO = USER_SERVICE.read(userTO.getKey());
1266         assertEquals("virtualvalue", userTO.getVirAttr("virtualdata").get().getValues().get(0));
1267     }
1268 
1269     @Test
1270     public void issueSYNCOPE313DB() throws Exception {
1271         // 1. create user in DB
1272         UserCR userCR = UserITCase.getUniqueSample("syncope313-db@syncope.apache.org");
1273         userCR.setPassword("security123");
1274         userCR.getResources().add(RESOURCE_NAME_TESTDB);
1275         UserTO user = createUser(userCR).getEntity();
1276         assertNotNull(user);
1277         assertFalse(user.getResources().isEmpty());
1278 
1279         // 2. Check that the DB resource has the correct password
1280         JdbcTemplate jdbcTemplate = new JdbcTemplate(testDataSource);
1281         String value = queryForObject(jdbcTemplate,
1282                 MAX_WAIT_SECONDS, "SELECT PASSWORD FROM test WHERE ID=?", String.class, user.getUsername());
1283         assertEquals(Encryptor.getInstance().encode("security123", CipherAlgorithm.SHA1), value.toUpperCase());
1284 
1285         // 3. Update the password in the DB
1286         String newCleanPassword = "new-security";
1287         String newPassword = Encryptor.getInstance().encode(newCleanPassword, CipherAlgorithm.SHA1);
1288         jdbcTemplate.execute("UPDATE test set PASSWORD='" + newPassword + "' where ID='" + user.getUsername() + '\'');
1289 
1290         // 4. Pull the user from the resource
1291         ImplementationTO pullActions = new ImplementationTO();
1292         pullActions.setKey(DBPasswordPullActions.class.getSimpleName());
1293         pullActions.setEngine(ImplementationEngine.JAVA);
1294         pullActions.setType(IdMImplementationType.PULL_ACTIONS);
1295         pullActions.setBody(DBPasswordPullActions.class.getName());
1296         Response response = IMPLEMENTATION_SERVICE.create(pullActions);
1297         pullActions = IMPLEMENTATION_SERVICE.read(
1298                 pullActions.getType(), response.getHeaderString(RESTHeaders.RESOURCE_KEY));
1299         assertNotNull(pullActions);
1300 
1301         PullTaskTO pullTask = new PullTaskTO();
1302         pullTask.setDestinationRealm(SyncopeConstants.ROOT_REALM);
1303         pullTask.setName("DB Pull Task");
1304         pullTask.setActive(true);
1305         pullTask.setPerformCreate(true);
1306         pullTask.setPerformUpdate(true);
1307         pullTask.setPullMode(PullMode.FULL_RECONCILIATION);
1308         pullTask.setResource(RESOURCE_NAME_TESTDB);
1309         pullTask.getActions().add(pullActions.getKey());
1310         Response taskResponse = TASK_SERVICE.create(TaskType.PULL, pullTask);
1311 
1312         PullTaskTO actual = getObject(taskResponse.getLocation(), TaskService.class, PullTaskTO.class);
1313         assertNotNull(actual);
1314 
1315         pullTask = TASK_SERVICE.read(TaskType.PULL, actual.getKey(), true);
1316         assertNotNull(pullTask);
1317         assertEquals(actual.getKey(), pullTask.getKey());
1318         assertEquals(actual.getJobDelegate(), pullTask.getJobDelegate());
1319 
1320         ExecTO execution = execProvisioningTask(
1321                 TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
1322         assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1323 
1324         // 5. Test the pulled user
1325         Triple<Map<String, Set<String>>, List<String>, UserTO> self =
1326                 CLIENT_FACTORY.create(user.getUsername(), newCleanPassword).self();
1327         assertNotNull(self);
1328 
1329         // 6. Delete PullTask + user
1330         TASK_SERVICE.delete(TaskType.PULL, pullTask.getKey());
1331         deleteUser(user.getKey());
1332     }
1333 
1334     @Test
1335     public void issueSYNCOPE313LDAP() throws Exception {
1336         // First of all, clear any potential conflict with existing user / group
1337         ldapCleanup();
1338 
1339         UserTO user = null;
1340         PullTaskTO pullTask = null;
1341         ConnInstanceTO resourceConnector = null;
1342         ConnConfProperty property = null;
1343         try {
1344             // 1. create user in LDAP
1345             String oldCleanPassword = "security123";
1346             UserCR userCR = UserITCase.getUniqueSample("syncope313-ldap@syncope.apache.org");
1347             userCR.setPassword(oldCleanPassword);
1348             userCR.getResources().add(RESOURCE_NAME_LDAP);
1349             user = createUser(userCR).getEntity();
1350             assertNotNull(user);
1351             assertFalse(user.getResources().isEmpty());
1352 
1353             // 2. request to change password only on Syncope and not on LDAP
1354             String newCleanPassword = "new-security123";
1355             UserUR userUR = new UserUR();
1356             userUR.setKey(user.getKey());
1357             userUR.setPassword(new PasswordPatch.Builder().value(newCleanPassword).build());
1358             user = updateUser(userUR).getEntity();
1359 
1360             // 3. Check that the Syncope user now has the changed password
1361             Triple<Map<String, Set<String>>, List<String>, UserTO> self =
1362                     CLIENT_FACTORY.create(user.getUsername(), newCleanPassword).self();
1363             assertNotNull(self);
1364 
1365             // 4. Check that the LDAP resource has the old password
1366             ConnObject connObject =
1367                     RESOURCE_SERVICE.readConnObject(RESOURCE_NAME_LDAP, AnyTypeKind.USER.name(), user.getKey());
1368             assertNotNull(getLdapRemoteObject(
1369                     connObject.getAttr(Name.NAME).get().getValues().get(0),
1370                     oldCleanPassword,
1371                     connObject.getAttr(Name.NAME).get().getValues().get(0)));
1372 
1373             // 5. Update the LDAP Connector to retrieve passwords
1374             ResourceTO ldapResource = RESOURCE_SERVICE.read(RESOURCE_NAME_LDAP);
1375             resourceConnector = CONNECTOR_SERVICE.read(
1376                     ldapResource.getConnector(), Locale.ENGLISH.getLanguage());
1377             property = resourceConnector.getConf("retrievePasswordsWithSearch").get();
1378             property.getValues().clear();
1379             property.getValues().add(Boolean.TRUE);
1380             CONNECTOR_SERVICE.update(resourceConnector);
1381 
1382             // 6. Pull the user from the resource
1383             ImplementationTO pullActions = new ImplementationTO();
1384             pullActions.setKey(LDAPPasswordPullActions.class.getSimpleName());
1385             pullActions.setEngine(ImplementationEngine.JAVA);
1386             pullActions.setType(IdMImplementationType.PULL_ACTIONS);
1387             pullActions.setBody(LDAPPasswordPullActions.class.getName());
1388             Response response = IMPLEMENTATION_SERVICE.create(pullActions);
1389             pullActions = IMPLEMENTATION_SERVICE.read(
1390                     pullActions.getType(), response.getHeaderString(RESTHeaders.RESOURCE_KEY));
1391             assertNotNull(pullActions);
1392 
1393             pullTask = new PullTaskTO();
1394             pullTask.setDestinationRealm(SyncopeConstants.ROOT_REALM);
1395             pullTask.setName(getUUIDString());
1396             pullTask.setActive(true);
1397             pullTask.setPerformCreate(true);
1398             pullTask.setPerformUpdate(true);
1399             pullTask.setPullMode(PullMode.FULL_RECONCILIATION);
1400             pullTask.setResource(RESOURCE_NAME_LDAP);
1401             pullTask.getActions().add(pullActions.getKey());
1402             Response taskResponse = TASK_SERVICE.create(TaskType.PULL, pullTask);
1403 
1404             pullTask = getObject(taskResponse.getLocation(), TaskService.class, PullTaskTO.class);
1405             assertNotNull(pullTask);
1406 
1407             ExecTO execution = execProvisioningTask(
1408                     TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
1409             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1410 
1411             // 7. Test the pulled user
1412             self = CLIENT_FACTORY.create(user.getUsername(), oldCleanPassword).self();
1413             assertNotNull(self);
1414         } catch (Exception e) {
1415             fail(e::getMessage);
1416         } finally {
1417             // Delete PullTask + user + reset the connector
1418             if (pullTask != null && pullTask.getKey() != null) {
1419                 TASK_SERVICE.delete(TaskType.PULL, pullTask.getKey());
1420             }
1421 
1422             if (resourceConnector != null && property != null) {
1423                 property.getValues().clear();
1424                 property.getValues().add(Boolean.FALSE);
1425                 CONNECTOR_SERVICE.update(resourceConnector);
1426             }
1427 
1428             if (user != null) {
1429                 deleteUser(user.getKey());
1430             }
1431         }
1432     }
1433 
1434     @Test
1435     public void issueSYNCOPE1062() {
1436         GroupTO propagationGroup = null;
1437         PullTaskTO pullTask = null;
1438         UserTO user = null;
1439         GroupTO group = null;
1440         try {
1441             // 1. create group with resource for propagation
1442             GroupCR propagationGroupCR = GroupITCase.getBasicSample("SYNCOPE1062");
1443             propagationGroupCR.getResources().add(RESOURCE_NAME_DBPULL);
1444             propagationGroup = createGroup(propagationGroupCR).getEntity();
1445 
1446             // 2. create pull task for another resource, with user template assigning the group above
1447             pullTask = new PullTaskTO();
1448             pullTask.setDestinationRealm(SyncopeConstants.ROOT_REALM);
1449             pullTask.setName("SYNCOPE1062");
1450             pullTask.setActive(true);
1451             pullTask.setPerformCreate(true);
1452             pullTask.setPerformUpdate(true);
1453             pullTask.setPullMode(PullMode.FULL_RECONCILIATION);
1454             pullTask.setResource(RESOURCE_NAME_LDAP);
1455 
1456             UserTO template = new UserTO();
1457             template.getAuxClasses().add("minimal group");
1458             template.getMemberships().add(new MembershipTO.Builder(propagationGroup.getKey()).build());
1459             template.getPlainAttrs().add(attr("firstname", "'fixed'"));
1460             pullTask.getTemplates().put(AnyTypeKind.USER.name(), template);
1461 
1462             Response taskResponse = TASK_SERVICE.create(TaskType.PULL, pullTask);
1463             pullTask = getObject(taskResponse.getLocation(), TaskService.class, PullTaskTO.class);
1464             assertNotNull(pullTask);
1465             assertFalse(pullTask.getTemplates().isEmpty());
1466 
1467             // 3. exec the pull task
1468             ExecTO execution = execProvisioningTask(
1469                     TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
1470             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1471 
1472             // the user is successfully pulled...
1473             user = USER_SERVICE.read("pullFromLDAP");
1474             assertNotNull(user);
1475             assertEquals("pullFromLDAP@syncope.apache.org", user.getPlainAttr("email").get().getValues().get(0));
1476 
1477             group = GROUP_SERVICE.read("testLDAPGroup");
1478             assertNotNull(group);
1479 
1480             ConnObject connObject =
1481                     RESOURCE_SERVICE.readConnObject(RESOURCE_NAME_LDAP, AnyTypeKind.USER.name(), user.getKey());
1482             assertNotNull(connObject);
1483             assertEquals("pullFromLDAP@syncope.apache.org", connObject.getAttr("mail").get().getValues().get(0));
1484             Attr userDn = connObject.getAttr(Name.NAME).get();
1485             assertNotNull(userDn);
1486             assertEquals(1, userDn.getValues().size());
1487             assertNotNull(
1488                     getLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD, userDn.getValues().get(0)));
1489 
1490             // ...and propagated
1491             PagedResult<TaskTO> propagationTasks = TASK_SERVICE.search(new TaskQuery.Builder(TaskType.PROPAGATION).
1492                     resource(RESOURCE_NAME_DBPULL).
1493                     anyTypeKind(AnyTypeKind.USER).entityKey(user.getKey()).build());
1494             assertEquals(1, propagationTasks.getSize());
1495 
1496             // 4. update the user on the external resource
1497             updateLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD,
1498                     userDn.getValues().get(0), Map.of("mail", "pullFromLDAP2@syncope.apache.org"));
1499 
1500             connObject = RESOURCE_SERVICE.readConnObject(RESOURCE_NAME_LDAP, AnyTypeKind.USER.name(), user.getKey());
1501             assertNotNull(connObject);
1502             assertEquals("pullFromLDAP2@syncope.apache.org", connObject.getAttr("mail").get().getValues().get(0));
1503 
1504             // 5. exec the pull task again
1505             execution = execProvisioningTask(TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
1506             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1507 
1508             // the internal is updated...
1509             user = USER_SERVICE.read("pullFromLDAP");
1510             assertNotNull(user);
1511             assertEquals("pullFromLDAP2@syncope.apache.org", user.getPlainAttr("email").get().getValues().get(0));
1512 
1513             // ...and propagated
1514             propagationTasks = TASK_SERVICE.search(new TaskQuery.Builder(TaskType.PROPAGATION).
1515                     resource(RESOURCE_NAME_DBPULL).
1516                     anyTypeKind(AnyTypeKind.USER).entityKey(user.getKey()).build());
1517             assertEquals(2, propagationTasks.getSize());
1518         } catch (Exception e) {
1519             LOG.error("Unexpected during issueSYNCOPE1062()", e);
1520             fail(e::getMessage);
1521         } finally {
1522             if (pullTask != null) {
1523                 TASK_SERVICE.delete(TaskType.PULL, pullTask.getKey());
1524             }
1525 
1526             if (propagationGroup != null) {
1527                 GROUP_SERVICE.delete(propagationGroup.getKey());
1528             }
1529 
1530             if (group != null) {
1531                 GROUP_SERVICE.delete(group.getKey());
1532             }
1533             if (user != null) {
1534                 USER_SERVICE.delete(user.getKey());
1535             }
1536         }
1537     }
1538 
1539     @Test
1540     public void issueSYNCOPE1656() throws NamingException {
1541         // preliminary create a new LDAP object
1542         createLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD, prepareLdapAttributes(
1543                 "pullFromLDAP_issue1656",
1544                 "pullFromLDAP_issue1656@syncope.apache.org",
1545                 "Active",
1546                 "pullFromLDAP_issue1656",
1547                 "Surname",
1548                 "5BAA61E4C9B93F3F0682250B6CF8331B7EE68FD8",
1549                 "odd",
1550                 "password"));
1551         try {
1552             // 1. Pull from resource-ldap
1553             PullTaskTO pullTask = new PullTaskTO();
1554             pullTask.setDestinationRealm(SyncopeConstants.ROOT_REALM);
1555             pullTask.setName("SYNCOPE1656");
1556             pullTask.setActive(true);
1557             pullTask.setPerformCreate(true);
1558             pullTask.setPerformUpdate(true);
1559             pullTask.setRemediation(true);
1560             pullTask.setPullMode(PullMode.FULL_RECONCILIATION);
1561             pullTask.setResource(RESOURCE_NAME_LDAP);
1562 
1563             Response taskResponse = TASK_SERVICE.create(TaskType.PULL, pullTask);
1564             pullTask = getObject(taskResponse.getLocation(), TaskService.class, PullTaskTO.class);
1565             assertNotNull(pullTask);
1566 
1567             ExecTO execution = execProvisioningTask(
1568                     TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
1569             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1570 
1571             UserTO pullFromLDAP4issue1656 = USER_SERVICE.read("pullFromLDAP_issue1656");
1572             assertEquals("pullFromLDAP_issue1656@syncope.apache.org",
1573                     pullFromLDAP4issue1656.getPlainAttr("email").get().getValues().get(0));
1574             // 2. Edit mail attribute directly on the resource in order to have a not valid email
1575             ConnObject connObject = RESOURCE_SERVICE.readConnObject(
1576                     RESOURCE_NAME_LDAP, AnyTypeKind.USER.name(), pullFromLDAP4issue1656.getKey());
1577             assertNotNull(connObject);
1578             assertEquals("pullFromLDAP_issue1656@syncope.apache.org",
1579                     connObject.getAttr("mail").get().getValues().get(0));
1580             Attr userDn = connObject.getAttr(Name.NAME).get();
1581             assertNotNull(userDn);
1582             assertEquals(1, userDn.getValues().size());
1583             updateLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD,
1584                     userDn.getValues().get(0), Collections.singletonMap("mail", "pullFromLDAP_issue1656@"));
1585             // 3. Pull again from resource-ldap
1586             execution = execProvisioningTask(TASK_SERVICE, TaskType.PULL, pullTask.getKey(), MAX_WAIT_SECONDS, false);
1587             assertEquals(ExecStatus.SUCCESS, ExecStatus.valueOf(execution.getStatus()));
1588             assertTrue(execution.getMessage().contains("UPDATE FAILURE"));
1589             pullFromLDAP4issue1656 = USER_SERVICE.read("pullFromLDAP_issue1656");
1590             assertEquals("pullFromLDAP_issue1656@syncope.apache.org",
1591                     pullFromLDAP4issue1656.getPlainAttr("email").get().getValues().get(0));
1592             String pullFromLDAP4issue1656Key = pullFromLDAP4issue1656.getKey();
1593             // a remediation for pullFromLDAP_issue1656 email should have been created
1594             PagedResult<RemediationTO> remediations =
1595                     REMEDIATION_SERVICE.list(new RemediationQuery.Builder().page(1).size(10).build());
1596             assertTrue(remediations.getResult().stream().filter(r -> r.getAnyURPayload() != null).anyMatch(
1597                     r -> pullFromLDAP4issue1656Key.equals(r.getAnyURPayload().getKey())));
1598             assertTrue(remediations.getResult().stream().anyMatch(r -> StringUtils.contains(r.getError(),
1599                     "\"pullFromLDAP_issue1656@\" is not a valid email address")));
1600         } finally {
1601             // remove test entity
1602             removeLdapRemoteObject(RESOURCE_LDAP_ADMIN_DN, RESOURCE_LDAP_ADMIN_PWD,
1603                     "uid=pullFromLDAP_issue1656,ou=People,o=isp");
1604             cleanUpRemediations();
1605         }
1606     }
1607 
1608     private static void cleanUpRemediations() {
1609         REMEDIATION_SERVICE.list(new RemediationQuery.Builder().page(1).size(100).build()).getResult().forEach(
1610                 r -> REMEDIATION_SERVICE.delete(r.getKey()));
1611     }
1612 }