001/* 002 * Copyright 2013-2019 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2013-2019 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.OutputStream; 026import java.util.Collections; 027import java.util.LinkedHashMap; 028import java.util.List; 029import java.util.Map; 030import java.util.TreeMap; 031import java.util.concurrent.atomic.AtomicLong; 032 033import com.unboundid.asn1.ASN1OctetString; 034import com.unboundid.ldap.sdk.Attribute; 035import com.unboundid.ldap.sdk.DN; 036import com.unboundid.ldap.sdk.Filter; 037import com.unboundid.ldap.sdk.LDAPConnectionOptions; 038import com.unboundid.ldap.sdk.LDAPConnectionPool; 039import com.unboundid.ldap.sdk.LDAPException; 040import com.unboundid.ldap.sdk.LDAPSearchException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.SearchRequest; 043import com.unboundid.ldap.sdk.SearchResult; 044import com.unboundid.ldap.sdk.SearchResultEntry; 045import com.unboundid.ldap.sdk.SearchResultReference; 046import com.unboundid.ldap.sdk.SearchResultListener; 047import com.unboundid.ldap.sdk.SearchScope; 048import com.unboundid.ldap.sdk.Version; 049import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl; 050import com.unboundid.util.Debug; 051import com.unboundid.util.LDAPCommandLineTool; 052import com.unboundid.util.StaticUtils; 053import com.unboundid.util.ThreadSafety; 054import com.unboundid.util.ThreadSafetyLevel; 055import com.unboundid.util.args.ArgumentException; 056import com.unboundid.util.args.ArgumentParser; 057import com.unboundid.util.args.DNArgument; 058import com.unboundid.util.args.IntegerArgument; 059import com.unboundid.util.args.StringArgument; 060 061 062 063/** 064 * This class provides a tool that may be used to identify references to entries 065 * that do not exist. This tool can be useful for verifying existing data in 066 * directory servers that provide support for referential integrity. 067 * <BR><BR> 068 * All of the necessary information is provided using command line arguments. 069 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 070 * class, as well as the following additional arguments: 071 * <UL> 072 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use 073 * for the searches. At least one base DN must be provided.</LI> 074 * <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute 075 * that is expected to contain references to other entries. This 076 * attribute should be indexed for equality searches, and its values 077 * should be DNs. At least one attribute must be provided.</LI> 078 * <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search 079 * to find entries with references to other entries should use the simple 080 * paged results control to iterate across entries in fixed-size pages 081 * rather than trying to use a single search to identify all entries that 082 * reference other entries.</LI> 083 * </UL> 084 */ 085@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 086public final class IdentifyReferencesToMissingEntries 087 extends LDAPCommandLineTool 088 implements SearchResultListener 089{ 090 /** 091 * The serial version UID for this serializable class. 092 */ 093 private static final long serialVersionUID = 1981894839719501258L; 094 095 096 097 // The number of entries examined so far. 098 private final AtomicLong entriesExamined; 099 100 // The argument used to specify the base DNs to use for searches. 101 private DNArgument baseDNArgument; 102 103 // The argument used to specify the search page size. 104 private IntegerArgument pageSizeArgument; 105 106 // The connection to use for retrieving referenced entries. 107 private LDAPConnectionPool getReferencedEntriesPool; 108 109 // A map with counts of missing references by attribute type. 110 private final Map<String,AtomicLong> missingReferenceCounts; 111 112 // The names of the attributes for which to find missing references. 113 private String[] attributes; 114 115 // The argument used to specify the attributes for which to find missing 116 // references. 117 private StringArgument attributeArgument; 118 119 120 121 /** 122 * Parse the provided command line arguments and perform the appropriate 123 * processing. 124 * 125 * @param args The command line arguments provided to this program. 126 */ 127 public static void main(final String... args) 128 { 129 final ResultCode resultCode = main(args, System.out, System.err); 130 if (resultCode != ResultCode.SUCCESS) 131 { 132 System.exit(resultCode.intValue()); 133 } 134 } 135 136 137 138 /** 139 * Parse the provided command line arguments and perform the appropriate 140 * processing. 141 * 142 * @param args The command line arguments provided to this program. 143 * @param outStream The output stream to which standard out should be 144 * written. It may be {@code null} if output should be 145 * suppressed. 146 * @param errStream The output stream to which standard error should be 147 * written. It may be {@code null} if error messages 148 * should be suppressed. 149 * 150 * @return A result code indicating whether the processing was successful. 151 */ 152 public static ResultCode main(final String[] args, 153 final OutputStream outStream, 154 final OutputStream errStream) 155 { 156 final IdentifyReferencesToMissingEntries tool = 157 new IdentifyReferencesToMissingEntries(outStream, errStream); 158 return tool.runTool(args); 159 } 160 161 162 163 /** 164 * Creates a new instance of this tool. 165 * 166 * @param outStream The output stream to which standard out should be 167 * written. It may be {@code null} if output should be 168 * suppressed. 169 * @param errStream The output stream to which standard error should be 170 * written. It may be {@code null} if error messages 171 * should be suppressed. 172 */ 173 public IdentifyReferencesToMissingEntries(final OutputStream outStream, 174 final OutputStream errStream) 175 { 176 super(outStream, errStream); 177 178 baseDNArgument = null; 179 pageSizeArgument = null; 180 attributeArgument = null; 181 getReferencedEntriesPool = null; 182 183 entriesExamined = new AtomicLong(0L); 184 missingReferenceCounts = new TreeMap<>(); 185 } 186 187 188 189 /** 190 * Retrieves the name of this tool. It should be the name of the command used 191 * to invoke this tool. 192 * 193 * @return The name for this tool. 194 */ 195 @Override() 196 public String getToolName() 197 { 198 return "identify-references-to-missing-entries"; 199 } 200 201 202 203 /** 204 * Retrieves a human-readable description for this tool. 205 * 206 * @return A human-readable description for this tool. 207 */ 208 @Override() 209 public String getToolDescription() 210 { 211 return "This tool may be used to identify entries containing one or more " + 212 "attributes which reference entries that do not exist. This may " + 213 "require the ability to perform unindexed searches and/or the " + 214 "ability to use the simple paged results control."; 215 } 216 217 218 219 /** 220 * Retrieves a version string for this tool, if available. 221 * 222 * @return A version string for this tool, or {@code null} if none is 223 * available. 224 */ 225 @Override() 226 public String getToolVersion() 227 { 228 return Version.NUMERIC_VERSION_STRING; 229 } 230 231 232 233 /** 234 * Indicates whether this tool should provide support for an interactive mode, 235 * in which the tool offers a mode in which the arguments can be provided in 236 * a text-driven menu rather than requiring them to be given on the command 237 * line. If interactive mode is supported, it may be invoked using the 238 * "--interactive" argument. Alternately, if interactive mode is supported 239 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 240 * interactive mode may be invoked by simply launching the tool without any 241 * arguments. 242 * 243 * @return {@code true} if this tool supports interactive mode, or 244 * {@code false} if not. 245 */ 246 @Override() 247 public boolean supportsInteractiveMode() 248 { 249 return true; 250 } 251 252 253 254 /** 255 * Indicates whether this tool defaults to launching in interactive mode if 256 * the tool is invoked without any command-line arguments. This will only be 257 * used if {@link #supportsInteractiveMode()} returns {@code true}. 258 * 259 * @return {@code true} if this tool defaults to using interactive mode if 260 * launched without any command-line arguments, or {@code false} if 261 * not. 262 */ 263 @Override() 264 public boolean defaultsToInteractiveMode() 265 { 266 return true; 267 } 268 269 270 271 /** 272 * Indicates whether this tool should provide arguments for redirecting output 273 * to a file. If this method returns {@code true}, then the tool will offer 274 * an "--outputFile" argument that will specify the path to a file to which 275 * all standard output and standard error content will be written, and it will 276 * also offer a "--teeToStandardOut" argument that can only be used if the 277 * "--outputFile" argument is present and will cause all output to be written 278 * to both the specified output file and to standard output. 279 * 280 * @return {@code true} if this tool should provide arguments for redirecting 281 * output to a file, or {@code false} if not. 282 */ 283 @Override() 284 protected boolean supportsOutputFile() 285 { 286 return true; 287 } 288 289 290 291 /** 292 * Indicates whether this tool should default to interactively prompting for 293 * the bind password if a password is required but no argument was provided 294 * to indicate how to get the password. 295 * 296 * @return {@code true} if this tool should default to interactively 297 * prompting for the bind password, or {@code false} if not. 298 */ 299 @Override() 300 protected boolean defaultToPromptForBindPassword() 301 { 302 return true; 303 } 304 305 306 307 /** 308 * Indicates whether this tool supports the use of a properties file for 309 * specifying default values for arguments that aren't specified on the 310 * command line. 311 * 312 * @return {@code true} if this tool supports the use of a properties file 313 * for specifying default values for arguments that aren't specified 314 * on the command line, or {@code false} if not. 315 */ 316 @Override() 317 public boolean supportsPropertiesFile() 318 { 319 return true; 320 } 321 322 323 324 /** 325 * Indicates whether the LDAP-specific arguments should include alternate 326 * versions of all long identifiers that consist of multiple words so that 327 * they are available in both camelCase and dash-separated versions. 328 * 329 * @return {@code true} if this tool should provide multiple versions of 330 * long identifiers for LDAP-specific arguments, or {@code false} if 331 * not. 332 */ 333 @Override() 334 protected boolean includeAlternateLongIdentifiers() 335 { 336 return true; 337 } 338 339 340 341 /** 342 * Indicates whether this tool should provide a command-line argument that 343 * allows for low-level SSL debugging. If this returns {@code true}, then an 344 * "--enableSSLDebugging}" argument will be added that sets the 345 * "javax.net.debug" system property to "all" before attempting any 346 * communication. 347 * 348 * @return {@code true} if this tool should offer an "--enableSSLDebugging" 349 * argument, or {@code false} if not. 350 */ 351 @Override() 352 protected boolean supportsSSLDebugging() 353 { 354 return true; 355 } 356 357 358 359 /** 360 * Adds the arguments needed by this command-line tool to the provided 361 * argument parser which are not related to connecting or authenticating to 362 * the directory server. 363 * 364 * @param parser The argument parser to which the arguments should be added. 365 * 366 * @throws ArgumentException If a problem occurs while adding the arguments. 367 */ 368 @Override() 369 public void addNonLDAPArguments(final ArgumentParser parser) 370 throws ArgumentException 371 { 372 String description = "The search base DN(s) to use to find entries with " + 373 "references to other entries. At least one base DN must be " + 374 "specified."; 375 baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}", 376 description); 377 baseDNArgument.addLongIdentifier("base-dn", true); 378 parser.addArgument(baseDNArgument); 379 380 description = "The attribute(s) for which to find missing references. " + 381 "At least one attribute must be specified, and each attribute " + 382 "must be indexed for equality searches and have values which are DNs."; 383 attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}", 384 description); 385 parser.addArgument(attributeArgument); 386 387 description = "The maximum number of entries to retrieve at a time when " + 388 "attempting to find entries with references to other entries. This " + 389 "requires that the authenticated user have permission to use the " + 390 "simple paged results control, but it can avoid problems with the " + 391 "server sending entries too quickly for the client to handle. By " + 392 "default, the simple paged results control will not be used."; 393 pageSizeArgument = 394 new IntegerArgument('z', "simplePageSize", false, 1, "{num}", 395 description, 1, Integer.MAX_VALUE); 396 pageSizeArgument.addLongIdentifier("simple-page-size", true); 397 parser.addArgument(pageSizeArgument); 398 } 399 400 401 402 /** 403 * Retrieves the connection options that should be used for connections that 404 * are created with this command line tool. Subclasses may override this 405 * method to use a custom set of connection options. 406 * 407 * @return The connection options that should be used for connections that 408 * are created with this command line tool. 409 */ 410 @Override() 411 public LDAPConnectionOptions getConnectionOptions() 412 { 413 final LDAPConnectionOptions options = new LDAPConnectionOptions(); 414 415 options.setUseSynchronousMode(true); 416 options.setResponseTimeoutMillis(0L); 417 418 return options; 419 } 420 421 422 423 /** 424 * Performs the core set of processing for this tool. 425 * 426 * @return A result code that indicates whether the processing completed 427 * successfully. 428 */ 429 @Override() 430 public ResultCode doToolProcessing() 431 { 432 // Establish a connection to the target directory server to use for 433 // finding references to entries. 434 final LDAPConnectionPool findReferencesPool; 435 try 436 { 437 findReferencesPool = getConnectionPool(1, 1); 438 findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true); 439 } 440 catch (final LDAPException le) 441 { 442 Debug.debugException(le); 443 err("Unable to establish a connection to the directory server: ", 444 StaticUtils.getExceptionMessage(le)); 445 return le.getResultCode(); 446 } 447 448 try 449 { 450 // Establish a second connection to use for retrieving referenced entries. 451 try 452 { 453 getReferencedEntriesPool = getConnectionPool(1,1); 454 getReferencedEntriesPool. 455 setRetryFailedOperationsDueToInvalidConnections(true); 456 } 457 catch (final LDAPException le) 458 { 459 Debug.debugException(le); 460 err("Unable to establish a connection to the directory server: ", 461 StaticUtils.getExceptionMessage(le)); 462 return le.getResultCode(); 463 } 464 465 466 // Get the set of attributes for which to find missing references. 467 final List<String> attrList = attributeArgument.getValues(); 468 attributes = new String[attrList.size()]; 469 attrList.toArray(attributes); 470 471 472 // Construct a search filter that will be used to find all entries with 473 // references to other entries. 474 final Filter filter; 475 if (attributes.length == 1) 476 { 477 filter = Filter.createPresenceFilter(attributes[0]); 478 missingReferenceCounts.put(attributes[0], new AtomicLong(0L)); 479 } 480 else 481 { 482 final Filter[] orComps = new Filter[attributes.length]; 483 for (int i=0; i < attributes.length; i++) 484 { 485 orComps[i] = Filter.createPresenceFilter(attributes[i]); 486 missingReferenceCounts.put(attributes[i], new AtomicLong(0L)); 487 } 488 filter = Filter.createORFilter(orComps); 489 } 490 491 492 // Iterate across all of the search base DNs and perform searches to find 493 // missing references. 494 for (final DN baseDN : baseDNArgument.getValues()) 495 { 496 ASN1OctetString cookie = null; 497 do 498 { 499 final SearchRequest searchRequest = new SearchRequest(this, 500 baseDN.toString(), SearchScope.SUB, filter, attributes); 501 if (pageSizeArgument.isPresent()) 502 { 503 searchRequest.addControl(new SimplePagedResultsControl( 504 pageSizeArgument.getValue(), cookie, false)); 505 } 506 507 SearchResult searchResult; 508 try 509 { 510 searchResult = findReferencesPool.search(searchRequest); 511 } 512 catch (final LDAPSearchException lse) 513 { 514 Debug.debugException(lse); 515 try 516 { 517 searchResult = findReferencesPool.search(searchRequest); 518 } 519 catch (final LDAPSearchException lse2) 520 { 521 Debug.debugException(lse2); 522 searchResult = lse2.getSearchResult(); 523 } 524 } 525 526 if (searchResult.getResultCode() != ResultCode.SUCCESS) 527 { 528 err("An error occurred while attempting to search for missing " + 529 "references to entries below " + baseDN + ": " + 530 searchResult.getDiagnosticMessage()); 531 return searchResult.getResultCode(); 532 } 533 534 final SimplePagedResultsControl pagedResultsResponse; 535 try 536 { 537 pagedResultsResponse = SimplePagedResultsControl.get(searchResult); 538 } 539 catch (final LDAPException le) 540 { 541 Debug.debugException(le); 542 err("An error occurred while attempting to decode a simple " + 543 "paged results response control in the response to a " + 544 "search for entries below " + baseDN + ": " + 545 StaticUtils.getExceptionMessage(le)); 546 return le.getResultCode(); 547 } 548 549 if (pagedResultsResponse != null) 550 { 551 if (pagedResultsResponse.moreResultsToReturn()) 552 { 553 cookie = pagedResultsResponse.getCookie(); 554 } 555 else 556 { 557 cookie = null; 558 } 559 } 560 } 561 while (cookie != null); 562 } 563 564 565 // See if there were any missing references found. 566 boolean missingReferenceFound = false; 567 for (final Map.Entry<String,AtomicLong> e : 568 missingReferenceCounts.entrySet()) 569 { 570 final long numMissing = e.getValue().get(); 571 if (numMissing > 0L) 572 { 573 if (! missingReferenceFound) 574 { 575 err(); 576 missingReferenceFound = true; 577 } 578 579 err("Found " + numMissing + ' ' + e.getKey() + 580 " references to entries that do not exist."); 581 } 582 } 583 584 if (missingReferenceFound) 585 { 586 return ResultCode.CONSTRAINT_VIOLATION; 587 } 588 else 589 { 590 out("No references were found to entries that do not exist."); 591 return ResultCode.SUCCESS; 592 } 593 } 594 finally 595 { 596 findReferencesPool.close(); 597 598 if (getReferencedEntriesPool != null) 599 { 600 getReferencedEntriesPool.close(); 601 } 602 } 603 } 604 605 606 607 /** 608 * Retrieves a map that correlates the number of missing references found by 609 * attribute type. 610 * 611 * @return A map that correlates the number of missing references found by 612 * attribute type. 613 */ 614 public Map<String,AtomicLong> getMissingReferenceCounts() 615 { 616 return Collections.unmodifiableMap(missingReferenceCounts); 617 } 618 619 620 621 /** 622 * Retrieves a set of information that may be used to generate example usage 623 * information. Each element in the returned map should consist of a map 624 * between an example set of arguments and a string that describes the 625 * behavior of the tool when invoked with that set of arguments. 626 * 627 * @return A set of information that may be used to generate example usage 628 * information. It may be {@code null} or empty if no example usage 629 * information is available. 630 */ 631 @Override() 632 public LinkedHashMap<String[],String> getExampleUsages() 633 { 634 final LinkedHashMap<String[],String> exampleMap = 635 new LinkedHashMap<>(StaticUtils.computeMapCapacity(1)); 636 637 final String[] args = 638 { 639 "--hostname", "server.example.com", 640 "--port", "389", 641 "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com", 642 "--bindPassword", "password", 643 "--baseDN", "dc=example,dc=com", 644 "--attribute", "member", 645 "--attribute", "uniqueMember", 646 "--simplePageSize", "100" 647 }; 648 exampleMap.put(args, 649 "Identify all entries below dc=example,dc=com in which either the " + 650 "member or uniqueMember attribute references an entry that " + 651 "does not exist."); 652 653 return exampleMap; 654 } 655 656 657 658 /** 659 * Indicates that the provided search result entry has been returned by the 660 * server and may be processed by this search result listener. 661 * 662 * @param searchEntry The search result entry that has been returned by the 663 * server. 664 */ 665 @Override() 666 public void searchEntryReturned(final SearchResultEntry searchEntry) 667 { 668 try 669 { 670 // Find attributes which references to entries that do not exist. 671 for (final String attr : attributes) 672 { 673 final List<Attribute> attrList = 674 searchEntry.getAttributesWithOptions(attr, null); 675 for (final Attribute a : attrList) 676 { 677 for (final String value : a.getValues()) 678 { 679 try 680 { 681 final SearchResultEntry e = 682 getReferencedEntriesPool.getEntry(value, "1.1"); 683 if (e == null) 684 { 685 err("Entry '", searchEntry.getDN(), "' includes attribute ", 686 a.getName(), " that references entry '", value, 687 "' which does not exist."); 688 missingReferenceCounts.get(attr).incrementAndGet(); 689 } 690 } 691 catch (final LDAPException le) 692 { 693 Debug.debugException(le); 694 err("An error occurred while attempting to determine whether " + 695 "entry '" + value + "' referenced in attribute " + 696 a.getName() + " of entry '" + searchEntry.getDN() + 697 "' exists: " + StaticUtils.getExceptionMessage(le)); 698 missingReferenceCounts.get(attr).incrementAndGet(); 699 } 700 } 701 } 702 } 703 } 704 finally 705 { 706 final long count = entriesExamined.incrementAndGet(); 707 if ((count % 1000L) == 0L) 708 { 709 out(count, " entries examined"); 710 } 711 } 712 } 713 714 715 716 /** 717 * Indicates that the provided search result reference has been returned by 718 * the server and may be processed by this search result listener. 719 * 720 * @param searchReference The search result reference that has been returned 721 * by the server. 722 */ 723 @Override() 724 public void searchReferenceReturned( 725 final SearchResultReference searchReference) 726 { 727 // No implementation is required. This tool will not follow referrals. 728 } 729}