001/* 002 * Copyright 2008-2018 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2018 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.examples; 022 023 024 025import java.io.File; 026import java.io.FileInputStream; 027import java.io.InputStream; 028import java.io.IOException; 029import java.io.OutputStream; 030import java.util.ArrayList; 031import java.util.Iterator; 032import java.util.TreeMap; 033import java.util.LinkedHashMap; 034import java.util.List; 035import java.util.concurrent.atomic.AtomicLong; 036import java.util.zip.GZIPInputStream; 037 038import com.unboundid.ldap.sdk.Entry; 039import com.unboundid.ldap.sdk.LDAPConnection; 040import com.unboundid.ldap.sdk.LDAPException; 041import com.unboundid.ldap.sdk.ResultCode; 042import com.unboundid.ldap.sdk.Version; 043import com.unboundid.ldap.sdk.schema.Schema; 044import com.unboundid.ldap.sdk.schema.EntryValidator; 045import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 046import com.unboundid.ldif.DuplicateValueBehavior; 047import com.unboundid.ldif.LDIFException; 048import com.unboundid.ldif.LDIFReader; 049import com.unboundid.ldif.LDIFReaderEntryTranslator; 050import com.unboundid.ldif.LDIFWriter; 051import com.unboundid.util.Debug; 052import com.unboundid.util.LDAPCommandLineTool; 053import com.unboundid.util.StaticUtils; 054import com.unboundid.util.ThreadSafety; 055import com.unboundid.util.ThreadSafetyLevel; 056import com.unboundid.util.args.ArgumentException; 057import com.unboundid.util.args.ArgumentParser; 058import com.unboundid.util.args.BooleanArgument; 059import com.unboundid.util.args.FileArgument; 060import com.unboundid.util.args.IntegerArgument; 061import com.unboundid.util.args.StringArgument; 062 063 064 065/** 066 * This class provides a simple tool that can be used to validate that the 067 * contents of an LDIF file are valid. This includes ensuring that the contents 068 * can be parsed as valid LDIF, and it can also ensure that the LDIF content 069 * conforms to the server schema. It will obtain the schema by connecting to 070 * the server and retrieving the default schema (i.e., the schema which governs 071 * the root DSE). By default, a thorough set of validation will be performed, 072 * but it is possible to disable certain types of validation. 073 * <BR><BR> 074 * Some of the APIs demonstrated by this example include: 075 * <UL> 076 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 077 * package)</LI> 078 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 079 * package)</LI> 080 * <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI> 081 * <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema} 082 * package)</LI> 083 * </UL> 084 * <BR><BR> 085 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 086 * class (to obtain the information to use to connect to the server to read the 087 * schema), as well as the following additional arguments: 088 * <UL> 089 * <LI>"--schemaDirectory {path}" -- specifies the path to a directory 090 * containing files with schema definitions. If this argument is 091 * provided, then no attempt will be made to communicate with a directory 092 * server.</LI> 093 * <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF 094 * file to be validated.</LI> 095 * <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is 096 * compressed.</LI> 097 * <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file 098 * to be written with information about all entries that failed 099 * validation.</LI> 100 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 101 * concurrent threads to use when processing the LDIF. If this is not 102 * provided, then a default of one thread will be used.</LI> 103 * <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation 104 * process should ignore validation failures due to entries that contain 105 * object classes not defined in the server schema.</LI> 106 * <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process 107 * should ignore validation failures due to entries that contain 108 * attributes not defined in the server schema.</LI> 109 * <LI>"--ignoreMalformedDNs" -- indicates that the validation process should 110 * ignore validation failures due to entries with malformed DNs.</LI> 111 * <LI>"--ignoreMissingRDNValues" -- indicates that the validation process 112 * should ignore validation failures due to entries that contain an RDN 113 * attribute value that is not present in the set of entry 114 * attributes.</LI> 115 * <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation 116 * process should ignore validation failures due to entries that either do 117 * not have a structural object class or that have multiple structural 118 * object classes.</LI> 119 * <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation 120 * process should ignore validation failures due to entries containing 121 * auxiliary classes that are not allowed by a DIT content rule, or 122 * abstract classes that are not subclassed by an auxiliary or structural 123 * class contained in the entry.</LI> 124 * <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process 125 * should ignore validation failures due to entries including attributes 126 * that are not allowed or are explicitly prohibited by a DIT content 127 * rule.</LI> 128 * <LI>"--ignoreMissingAttributes" -- indicates that the validation process 129 * should ignore validation failures due to entries missing required 130 * attributes.</LI> 131 * <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation 132 * process should ignore validation failures due to single-valued 133 * attributes containing multiple values.</LI> 134 * <LI>"--ignoreAttributeSyntax" -- indicates that the validation process 135 * should ignore validation failures due to attribute values which violate 136 * the associated attribute syntax.</LI> 137 * <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation 138 * process should ignore validation failures due to attribute values which 139 * violate the associated attribute syntax, but only for the specified 140 * attribute types.</LI> 141 * <LI>"--ignoreNameForms" -- indicates that the validation process should 142 * ignore validation failures due to name form violations (in which the 143 * entry's RDN does not comply with the associated name form).</LI> 144 * </UL> 145 */ 146@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 147public final class ValidateLDIF 148 extends LDAPCommandLineTool 149 implements LDIFReaderEntryTranslator 150{ 151 /** 152 * The end-of-line character for this platform. 153 */ 154 private static final String EOL = System.getProperty("line.separator", "\n"); 155 156 157 158 // The arguments used by this program. 159 private BooleanArgument ignoreDuplicateValues; 160 private BooleanArgument ignoreUndefinedObjectClasses; 161 private BooleanArgument ignoreUndefinedAttributes; 162 private BooleanArgument ignoreMalformedDNs; 163 private BooleanArgument ignoreMissingRDNValues; 164 private BooleanArgument ignoreMissingSuperiorObjectClasses; 165 private BooleanArgument ignoreStructuralObjectClasses; 166 private BooleanArgument ignoreProhibitedObjectClasses; 167 private BooleanArgument ignoreProhibitedAttributes; 168 private BooleanArgument ignoreMissingAttributes; 169 private BooleanArgument ignoreSingleValuedAttributes; 170 private BooleanArgument ignoreAttributeSyntax; 171 private BooleanArgument ignoreNameForms; 172 private BooleanArgument isCompressed; 173 private FileArgument schemaDirectory; 174 private FileArgument ldifFile; 175 private FileArgument rejectFile; 176 private FileArgument encryptionPassphraseFile; 177 private IntegerArgument numThreads; 178 private StringArgument ignoreSyntaxViolationsForAttribute; 179 180 // The counter used to keep track of the number of entries processed. 181 private final AtomicLong entriesProcessed = new AtomicLong(0L); 182 183 // The counter used to keep track of the number of entries that could not be 184 // parsed as valid entries. 185 private final AtomicLong malformedEntries = new AtomicLong(0L); 186 187 // The entry validator that will be used to validate the entries. 188 private EntryValidator entryValidator; 189 190 // The LDIF writer that will be used to write rejected entries. 191 private LDIFWriter rejectWriter; 192 193 194 195 /** 196 * Parse the provided command line arguments and make the appropriate set of 197 * changes. 198 * 199 * @param args The command line arguments provided to this program. 200 */ 201 public static void main(final String[] args) 202 { 203 final ResultCode resultCode = main(args, System.out, System.err); 204 if (resultCode != ResultCode.SUCCESS) 205 { 206 System.exit(resultCode.intValue()); 207 } 208 } 209 210 211 212 /** 213 * Parse the provided command line arguments and make the appropriate set of 214 * changes. 215 * 216 * @param args The command line arguments provided to this program. 217 * @param outStream The output stream to which standard out should be 218 * written. It may be {@code null} if output should be 219 * suppressed. 220 * @param errStream The output stream to which standard error should be 221 * written. It may be {@code null} if error messages 222 * should be suppressed. 223 * 224 * @return A result code indicating whether the processing was successful. 225 */ 226 public static ResultCode main(final String[] args, 227 final OutputStream outStream, 228 final OutputStream errStream) 229 { 230 final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream); 231 return validateLDIF.runTool(args); 232 } 233 234 235 236 /** 237 * Creates a new instance of this tool. 238 * 239 * @param outStream The output stream to which standard out should be 240 * written. It may be {@code null} if output should be 241 * suppressed. 242 * @param errStream The output stream to which standard error should be 243 * written. It may be {@code null} if error messages 244 * should be suppressed. 245 */ 246 public ValidateLDIF(final OutputStream outStream, 247 final OutputStream errStream) 248 { 249 super(outStream, errStream); 250 } 251 252 253 254 /** 255 * Retrieves the name for this tool. 256 * 257 * @return The name for this tool. 258 */ 259 @Override() 260 public String getToolName() 261 { 262 return "validate-ldif"; 263 } 264 265 266 267 /** 268 * Retrieves the description for this tool. 269 * 270 * @return The description for this tool. 271 */ 272 @Override() 273 public String getToolDescription() 274 { 275 return "Validate the contents of an LDIF file " + 276 "against the server schema."; 277 } 278 279 280 281 /** 282 * Retrieves the version string for this tool. 283 * 284 * @return The version string for this tool. 285 */ 286 @Override() 287 public String getToolVersion() 288 { 289 return Version.NUMERIC_VERSION_STRING; 290 } 291 292 293 294 /** 295 * Indicates whether this tool should provide support for an interactive mode, 296 * in which the tool offers a mode in which the arguments can be provided in 297 * a text-driven menu rather than requiring them to be given on the command 298 * line. If interactive mode is supported, it may be invoked using the 299 * "--interactive" argument. Alternately, if interactive mode is supported 300 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 301 * interactive mode may be invoked by simply launching the tool without any 302 * arguments. 303 * 304 * @return {@code true} if this tool supports interactive mode, or 305 * {@code false} if not. 306 */ 307 @Override() 308 public boolean supportsInteractiveMode() 309 { 310 return true; 311 } 312 313 314 315 /** 316 * Indicates whether this tool defaults to launching in interactive mode if 317 * the tool is invoked without any command-line arguments. This will only be 318 * used if {@link #supportsInteractiveMode()} returns {@code true}. 319 * 320 * @return {@code true} if this tool defaults to using interactive mode if 321 * launched without any command-line arguments, or {@code false} if 322 * not. 323 */ 324 @Override() 325 public boolean defaultsToInteractiveMode() 326 { 327 return true; 328 } 329 330 331 332 /** 333 * Indicates whether this tool should provide arguments for redirecting output 334 * to a file. If this method returns {@code true}, then the tool will offer 335 * an "--outputFile" argument that will specify the path to a file to which 336 * all standard output and standard error content will be written, and it will 337 * also offer a "--teeToStandardOut" argument that can only be used if the 338 * "--outputFile" argument is present and will cause all output to be written 339 * to both the specified output file and to standard output. 340 * 341 * @return {@code true} if this tool should provide arguments for redirecting 342 * output to a file, or {@code false} if not. 343 */ 344 @Override() 345 protected boolean supportsOutputFile() 346 { 347 return true; 348 } 349 350 351 352 /** 353 * Indicates whether this tool should default to interactively prompting for 354 * the bind password if a password is required but no argument was provided 355 * to indicate how to get the password. 356 * 357 * @return {@code true} if this tool should default to interactively 358 * prompting for the bind password, or {@code false} if not. 359 */ 360 @Override() 361 protected boolean defaultToPromptForBindPassword() 362 { 363 return true; 364 } 365 366 367 368 /** 369 * Indicates whether this tool supports the use of a properties file for 370 * specifying default values for arguments that aren't specified on the 371 * command line. 372 * 373 * @return {@code true} if this tool supports the use of a properties file 374 * for specifying default values for arguments that aren't specified 375 * on the command line, or {@code false} if not. 376 */ 377 @Override() 378 public boolean supportsPropertiesFile() 379 { 380 return true; 381 } 382 383 384 385 /** 386 * Indicates whether the LDAP-specific arguments should include alternate 387 * versions of all long identifiers that consist of multiple words so that 388 * they are available in both camelCase and dash-separated versions. 389 * 390 * @return {@code true} if this tool should provide multiple versions of 391 * long identifiers for LDAP-specific arguments, or {@code false} if 392 * not. 393 */ 394 @Override() 395 protected boolean includeAlternateLongIdentifiers() 396 { 397 return true; 398 } 399 400 401 402 /** 403 * Adds the arguments used by this program that aren't already provided by the 404 * generic {@code LDAPCommandLineTool} framework. 405 * 406 * @param parser The argument parser to which the arguments should be added. 407 * 408 * @throws ArgumentException If a problem occurs while adding the arguments. 409 */ 410 @Override() 411 public void addNonLDAPArguments(final ArgumentParser parser) 412 throws ArgumentException 413 { 414 String description = "The path to the LDIF file to process. The tool " + 415 "will automatically attempt to detect whether the file is " + 416 "encrypted or compressed."; 417 ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description, 418 true, true, true, false); 419 ldifFile.addLongIdentifier("ldif-file", true); 420 parser.addArgument(ldifFile); 421 422 423 // Add an argument that makes it possible to read a compressed LDIF file. 424 // Note that this argument is no longer needed for dealing with compressed 425 // files, since the tool will automatically detect whether a file is 426 // compressed. However, the argument is still provided for the purpose of 427 // backward compatibility. 428 description = "Indicates that the specified LDIF file is compressed " + 429 "using gzip compression."; 430 isCompressed = new BooleanArgument('c', "isCompressed", description); 431 isCompressed.addLongIdentifier("is-compressed", true); 432 isCompressed.setHidden(true); 433 parser.addArgument(isCompressed); 434 435 436 // Add an argument that indicates that the tool should read the encryption 437 // passphrase from a file. 438 description = "Indicates that the specified LDIF file is encrypted and " + 439 "that the encryption passphrase is contained in the specified " + 440 "file. If the LDIF data is encrypted and this argument is not " + 441 "provided, then the tool will interactively prompt for the " + 442 "encryption passphrase."; 443 encryptionPassphraseFile = new FileArgument(null, 444 "encryptionPassphraseFile", false, 1, null, description, true, true, 445 true, false); 446 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 447 true); 448 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 449 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 450 true); 451 parser.addArgument(encryptionPassphraseFile); 452 453 454 description = "The path to the file to which rejected entries should be " + 455 "written."; 456 rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}", 457 description, false, true, true, false); 458 rejectFile.addLongIdentifier("reject-file", true); 459 parser.addArgument(rejectFile); 460 461 description = "The path to a directory containing one or more LDIF files " + 462 "with the schema information to use. If this is provided, " + 463 "then no LDAP communication will be performed."; 464 schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1, 465 "{path}", description, true, true, false, true); 466 schemaDirectory.addLongIdentifier("schema-directory", true); 467 parser.addArgument(schemaDirectory); 468 469 description = "The number of threads to use when processing the LDIF file."; 470 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 471 description, 1, Integer.MAX_VALUE, 1); 472 numThreads.addLongIdentifier("num-threads", true); 473 parser.addArgument(numThreads); 474 475 description = "Ignore validation failures due to entries containing " + 476 "duplicate values for the same attribute."; 477 ignoreDuplicateValues = 478 new BooleanArgument(null, "ignoreDuplicateValues", description); 479 ignoreDuplicateValues.setArgumentGroupName( 480 "Validation Strictness Arguments"); 481 ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true); 482 parser.addArgument(ignoreDuplicateValues); 483 484 description = "Ignore validation failures due to object classes not " + 485 "defined in the schema."; 486 ignoreUndefinedObjectClasses = 487 new BooleanArgument(null, "ignoreUndefinedObjectClasses", description); 488 ignoreUndefinedObjectClasses.setArgumentGroupName( 489 "Validation Strictness Arguments"); 490 ignoreUndefinedObjectClasses.addLongIdentifier( 491 "ignore-undefined-object-classes", true); 492 parser.addArgument(ignoreUndefinedObjectClasses); 493 494 description = "Ignore validation failures due to attributes not defined " + 495 "in the schema."; 496 ignoreUndefinedAttributes = 497 new BooleanArgument(null, "ignoreUndefinedAttributes", description); 498 ignoreUndefinedAttributes.setArgumentGroupName( 499 "Validation Strictness Arguments"); 500 ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes", 501 true); 502 parser.addArgument(ignoreUndefinedAttributes); 503 504 description = "Ignore validation failures due to entries with malformed " + 505 "DNs."; 506 ignoreMalformedDNs = 507 new BooleanArgument(null, "ignoreMalformedDNs", description); 508 ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments"); 509 ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true); 510 parser.addArgument(ignoreMalformedDNs); 511 512 description = "Ignore validation failures due to entries with RDN " + 513 "attribute values that are missing from the set of entry " + 514 "attributes."; 515 ignoreMissingRDNValues = 516 new BooleanArgument(null, "ignoreMissingRDNValues", description); 517 ignoreMissingRDNValues.setArgumentGroupName( 518 "Validation Strictness Arguments"); 519 ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true); 520 parser.addArgument(ignoreMissingRDNValues); 521 522 description = "Ignore validation failures due to entries without exactly " + 523 "structural object class."; 524 ignoreStructuralObjectClasses = 525 new BooleanArgument(null, "ignoreStructuralObjectClasses", 526 description); 527 ignoreStructuralObjectClasses.setArgumentGroupName( 528 "Validation Strictness Arguments"); 529 ignoreStructuralObjectClasses.addLongIdentifier( 530 "ignore-structural-object-classes", true); 531 parser.addArgument(ignoreStructuralObjectClasses); 532 533 description = "Ignore validation failures due to entries with object " + 534 "classes that are not allowed."; 535 ignoreProhibitedObjectClasses = 536 new BooleanArgument(null, "ignoreProhibitedObjectClasses", 537 description); 538 ignoreProhibitedObjectClasses.setArgumentGroupName( 539 "Validation Strictness Arguments"); 540 ignoreProhibitedObjectClasses.addLongIdentifier( 541 "ignore-prohibited-object-classes", true); 542 parser.addArgument(ignoreProhibitedObjectClasses); 543 544 description = "Ignore validation failures due to entries that are " + 545 "one or more superior object classes."; 546 ignoreMissingSuperiorObjectClasses = 547 new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses", 548 description); 549 ignoreMissingSuperiorObjectClasses.setArgumentGroupName( 550 "Validation Strictness Arguments"); 551 ignoreMissingSuperiorObjectClasses.addLongIdentifier( 552 "ignore-missing-superior-object-classes", true); 553 parser.addArgument(ignoreMissingSuperiorObjectClasses); 554 555 description = "Ignore validation failures due to entries with attributes " + 556 "that are not allowed."; 557 ignoreProhibitedAttributes = 558 new BooleanArgument(null, "ignoreProhibitedAttributes", description); 559 ignoreProhibitedAttributes.setArgumentGroupName( 560 "Validation Strictness Arguments"); 561 ignoreProhibitedAttributes.addLongIdentifier( 562 "ignore-prohibited-attributes", true); 563 parser.addArgument(ignoreProhibitedAttributes); 564 565 description = "Ignore validation failures due to entries missing " + 566 "required attributes."; 567 ignoreMissingAttributes = 568 new BooleanArgument(null, "ignoreMissingAttributes", description); 569 ignoreMissingAttributes.setArgumentGroupName( 570 "Validation Strictness Arguments"); 571 ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes", 572 true); 573 parser.addArgument(ignoreMissingAttributes); 574 575 description = "Ignore validation failures due to entries with multiple " + 576 "values for single-valued attributes."; 577 ignoreSingleValuedAttributes = 578 new BooleanArgument(null, "ignoreSingleValuedAttributes", description); 579 ignoreSingleValuedAttributes.setArgumentGroupName( 580 "Validation Strictness Arguments"); 581 ignoreSingleValuedAttributes.addLongIdentifier( 582 "ignore-single-valued-attributes", true); 583 parser.addArgument(ignoreSingleValuedAttributes); 584 585 description = "Ignore validation failures due to entries with attribute " + 586 "values that violate their associated syntax. If this is " + 587 "provided, then no attribute syntax violations will be " + 588 "flagged. If this is not provided, then all attribute " + 589 "syntax violations will be flagged except for violations " + 590 "in those attributes excluded by the " + 591 "--ignoreSyntaxViolationsForAttribute argument."; 592 ignoreAttributeSyntax = 593 new BooleanArgument(null, "ignoreAttributeSyntax", description); 594 ignoreAttributeSyntax.setArgumentGroupName( 595 "Validation Strictness Arguments"); 596 ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true); 597 parser.addArgument(ignoreAttributeSyntax); 598 599 description = "The name or OID of an attribute for which to ignore " + 600 "validation failures due to violations of the associated " + 601 "attribute syntax. This argument can only be used if the " + 602 "--ignoreAttributeSyntax argument is not provided."; 603 ignoreSyntaxViolationsForAttribute = new StringArgument(null, 604 "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description); 605 ignoreSyntaxViolationsForAttribute.setArgumentGroupName( 606 "Validation Strictness Arguments"); 607 ignoreSyntaxViolationsForAttribute.addLongIdentifier( 608 "ignore-syntax-violations-for-attribute", true); 609 parser.addArgument(ignoreSyntaxViolationsForAttribute); 610 611 description = "Ignore validation failures due to entries with RDNs " + 612 "that violate the associated name form definition."; 613 ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description); 614 ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments"); 615 ignoreNameForms.addLongIdentifier("ignore-name-forms", true); 616 parser.addArgument(ignoreNameForms); 617 618 619 // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments 620 // cannot be used together. 621 parser.addExclusiveArgumentSet(ignoreAttributeSyntax, 622 ignoreSyntaxViolationsForAttribute); 623 } 624 625 626 627 /** 628 * Performs the actual processing for this tool. In this case, it gets a 629 * connection to the directory server and uses it to retrieve the server 630 * schema. It then reads the LDIF file and validates each entry accordingly. 631 * 632 * @return The result code for the processing that was performed. 633 */ 634 @Override() 635 public ResultCode doToolProcessing() 636 { 637 // Get the connection to the directory server and use it to read the schema. 638 final Schema schema; 639 if (schemaDirectory.isPresent()) 640 { 641 final File schemaDir = schemaDirectory.getValue(); 642 643 try 644 { 645 final TreeMap<String,File> fileMap = new TreeMap<>(); 646 for (final File f : schemaDir.listFiles()) 647 { 648 final String name = f.getName(); 649 if (f.isFile() && name.endsWith(".ldif")) 650 { 651 fileMap.put(name, f); 652 } 653 } 654 655 if (fileMap.isEmpty()) 656 { 657 err("No LDIF files found in directory " + 658 schemaDir.getAbsolutePath()); 659 return ResultCode.PARAM_ERROR; 660 } 661 662 final ArrayList<File> fileList = new ArrayList<>(fileMap.values()); 663 schema = Schema.getSchema(fileList); 664 } 665 catch (final Exception e) 666 { 667 Debug.debugException(e); 668 err("Unable to read schema from files in directory " + 669 schemaDir.getAbsolutePath() + ": " + 670 StaticUtils.getExceptionMessage(e)); 671 return ResultCode.LOCAL_ERROR; 672 } 673 } 674 else 675 { 676 try 677 { 678 final LDAPConnection connection = getConnection(); 679 schema = connection.getSchema(); 680 connection.close(); 681 } 682 catch (final LDAPException le) 683 { 684 Debug.debugException(le); 685 err("Unable to connect to the directory server and read the schema: ", 686 le.getMessage()); 687 return le.getResultCode(); 688 } 689 } 690 691 692 // Get the encryption passphrase, if it was provided. 693 String encryptionPassphrase = null; 694 if (encryptionPassphraseFile.isPresent()) 695 { 696 try 697 { 698 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 699 encryptionPassphraseFile.getValue()); 700 } 701 catch (final LDAPException e) 702 { 703 Debug.debugException(e); 704 err(e.getMessage()); 705 return e.getResultCode(); 706 } 707 } 708 709 710 // Create the entry validator and initialize its configuration. 711 entryValidator = new EntryValidator(schema); 712 entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent()); 713 entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent()); 714 entryValidator.setCheckEntryMissingRDNValues( 715 !ignoreMissingRDNValues.isPresent()); 716 entryValidator.setCheckMissingAttributes( 717 !ignoreMissingAttributes.isPresent()); 718 entryValidator.setCheckNameForms(!ignoreNameForms.isPresent()); 719 entryValidator.setCheckProhibitedAttributes( 720 !ignoreProhibitedAttributes.isPresent()); 721 entryValidator.setCheckProhibitedObjectClasses( 722 !ignoreProhibitedObjectClasses.isPresent()); 723 entryValidator.setCheckMissingSuperiorObjectClasses( 724 !ignoreMissingSuperiorObjectClasses.isPresent()); 725 entryValidator.setCheckSingleValuedAttributes( 726 !ignoreSingleValuedAttributes.isPresent()); 727 entryValidator.setCheckStructuralObjectClasses( 728 !ignoreStructuralObjectClasses.isPresent()); 729 entryValidator.setCheckUndefinedAttributes( 730 !ignoreUndefinedAttributes.isPresent()); 731 entryValidator.setCheckUndefinedObjectClasses( 732 !ignoreUndefinedObjectClasses.isPresent()); 733 734 if (ignoreSyntaxViolationsForAttribute.isPresent()) 735 { 736 entryValidator.setIgnoreSyntaxViolationAttributeTypes( 737 ignoreSyntaxViolationsForAttribute.getValues()); 738 } 739 740 741 // Create an LDIF reader that can be used to read through the LDIF file. 742 final LDIFReader ldifReader; 743 rejectWriter = null; 744 try 745 { 746 InputStream inputStream = new FileInputStream(ldifFile.getValue()); 747 748 inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream( 749 inputStream, encryptionPassphrase, false, 750 "LDIF file '" + ldifFile.getValue().getPath() + 751 "' is encrypted. Please enter the encryption passphrase:", 752 "ERROR: The provided passphrase was incorrect.", 753 getOut(), getErr()).getFirst(); 754 755 if (isCompressed.isPresent()) 756 { 757 inputStream = new GZIPInputStream(inputStream); 758 } 759 else 760 { 761 inputStream = 762 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 763 } 764 765 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this); 766 } 767 catch (final Exception e) 768 { 769 Debug.debugException(e); 770 err("Unable to open the LDIF reader: ", 771 StaticUtils.getExceptionMessage(e)); 772 return ResultCode.LOCAL_ERROR; 773 } 774 775 ldifReader.setSchema(schema); 776 if (ignoreDuplicateValues.isPresent()) 777 { 778 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP); 779 } 780 else 781 { 782 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT); 783 } 784 785 try 786 { 787 // Create an LDIF writer that can be used to write information about 788 // rejected entries. 789 try 790 { 791 if (rejectFile.isPresent()) 792 { 793 rejectWriter = new LDIFWriter(rejectFile.getValue()); 794 } 795 } 796 catch (final Exception e) 797 { 798 Debug.debugException(e); 799 err("Unable to create the reject writer: ", 800 StaticUtils.getExceptionMessage(e)); 801 return ResultCode.LOCAL_ERROR; 802 } 803 804 ResultCode resultCode = ResultCode.SUCCESS; 805 while (true) 806 { 807 try 808 { 809 final Entry e = ldifReader.readEntry(); 810 if (e == null) 811 { 812 // Because we're performing parallel processing and returning null 813 // from the translate method, LDIFReader.readEntry() should never 814 // return a non-null value. However, it can throw an LDIFException 815 // if it encounters an invalid entry, or an IOException if there's 816 // a problem reading from the file, so we should still iterate 817 // through all of the entries to catch and report on those problems. 818 break; 819 } 820 } 821 catch (final LDIFException le) 822 { 823 Debug.debugException(le); 824 malformedEntries.incrementAndGet(); 825 826 if (resultCode == ResultCode.SUCCESS) 827 { 828 resultCode = ResultCode.DECODING_ERROR; 829 } 830 831 if (rejectWriter != null) 832 { 833 try 834 { 835 rejectWriter.writeComment( 836 "Unable to parse an entry read from LDIF:", false, false); 837 if (le.mayContinueReading()) 838 { 839 rejectWriter.writeComment( 840 StaticUtils.getExceptionMessage(le), false, true); 841 } 842 else 843 { 844 rejectWriter.writeComment( 845 StaticUtils.getExceptionMessage(le), false, 846 false); 847 rejectWriter.writeComment("Unable to continue LDIF processing.", 848 false, true); 849 err("Aborting LDIF processing: ", 850 StaticUtils.getExceptionMessage(le)); 851 return ResultCode.LOCAL_ERROR; 852 } 853 } 854 catch (final IOException ioe) 855 { 856 Debug.debugException(ioe); 857 err("Unable to write to the reject file:", 858 StaticUtils.getExceptionMessage(ioe)); 859 err("LDIF parse failure that triggered the rejection: ", 860 StaticUtils.getExceptionMessage(le)); 861 return ResultCode.LOCAL_ERROR; 862 } 863 } 864 } 865 catch (final IOException ioe) 866 { 867 Debug.debugException(ioe); 868 869 if (rejectWriter != null) 870 { 871 try 872 { 873 rejectWriter.writeComment("I/O error reading from LDIF:", false, 874 false); 875 rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe), 876 false, true); 877 return ResultCode.LOCAL_ERROR; 878 } 879 catch (final Exception ex) 880 { 881 Debug.debugException(ex); 882 err("I/O error reading from LDIF:", 883 StaticUtils.getExceptionMessage(ioe)); 884 return ResultCode.LOCAL_ERROR; 885 } 886 } 887 } 888 } 889 890 if (malformedEntries.get() > 0) 891 { 892 out(malformedEntries.get() + " entries were malformed and could not " + 893 "be read from the LDIF file."); 894 } 895 896 if (entryValidator.getInvalidEntries() > 0) 897 { 898 if (resultCode == ResultCode.SUCCESS) 899 { 900 resultCode = ResultCode.OBJECT_CLASS_VIOLATION; 901 } 902 903 for (final String s : entryValidator.getInvalidEntrySummary(true)) 904 { 905 out(s); 906 } 907 } 908 else 909 { 910 if (malformedEntries.get() == 0) 911 { 912 out("No errors were encountered."); 913 } 914 } 915 916 return resultCode; 917 } 918 finally 919 { 920 try 921 { 922 ldifReader.close(); 923 } 924 catch (final Exception e) 925 { 926 Debug.debugException(e); 927 } 928 929 try 930 { 931 if (rejectWriter != null) 932 { 933 rejectWriter.close(); 934 } 935 } 936 catch (final Exception e) 937 { 938 Debug.debugException(e); 939 } 940 } 941 } 942 943 944 945 /** 946 * Examines the provided entry to determine whether it conforms to the 947 * server schema. 948 * 949 * @param entry The entry to be examined. 950 * @param firstLineNumber The line number of the LDIF source on which the 951 * provided entry begins. 952 * 953 * @return The updated entry. This method will always return {@code null} 954 * because all of the real processing needed for the entry is 955 * performed in this method and the entry isn't needed any more 956 * after this method is done. 957 */ 958 @Override() 959 public Entry translate(final Entry entry, final long firstLineNumber) 960 { 961 final ArrayList<String> invalidReasons = new ArrayList<>(5); 962 if (! entryValidator.entryIsValid(entry, invalidReasons)) 963 { 964 if (rejectWriter != null) 965 { 966 synchronized (this) 967 { 968 try 969 { 970 rejectWriter.writeEntry(entry, listToString(invalidReasons)); 971 } 972 catch (final IOException ioe) 973 { 974 Debug.debugException(ioe); 975 } 976 } 977 } 978 } 979 980 final long numEntries = entriesProcessed.incrementAndGet(); 981 if ((numEntries % 1000L) == 0L) 982 { 983 out("Processed ", numEntries, " entries."); 984 } 985 986 return null; 987 } 988 989 990 991 /** 992 * Converts the provided list of strings into a single string. It will 993 * contain line breaks after all but the last element. 994 * 995 * @param l The list of strings to convert to a single string. 996 * 997 * @return The string from the provided list, or {@code null} if the provided 998 * list is empty or {@code null}. 999 */ 1000 private static String listToString(final List<String> l) 1001 { 1002 if ((l == null) || (l.isEmpty())) 1003 { 1004 return null; 1005 } 1006 1007 final StringBuilder buffer = new StringBuilder(); 1008 final Iterator<String> iterator = l.iterator(); 1009 while (iterator.hasNext()) 1010 { 1011 buffer.append(iterator.next()); 1012 if (iterator.hasNext()) 1013 { 1014 buffer.append(EOL); 1015 } 1016 } 1017 1018 return buffer.toString(); 1019 } 1020 1021 1022 1023 /** 1024 * {@inheritDoc} 1025 */ 1026 @Override() 1027 public LinkedHashMap<String[],String> getExampleUsages() 1028 { 1029 final LinkedHashMap<String[],String> examples = new LinkedHashMap<>(2); 1030 1031 String[] args = 1032 { 1033 "--hostname", "server.example.com", 1034 "--port", "389", 1035 "--ldifFile", "data.ldif", 1036 "--rejectFile", "rejects.ldif", 1037 "--numThreads", "4" 1038 }; 1039 String description = 1040 "Validate the contents of the 'data.ldif' file using the schema " + 1041 "defined in the specified directory server using four concurrent " + 1042 "threads. All types of validation will be performed, and " + 1043 "information about any errors will be written to the 'rejects.ldif' " + 1044 "file."; 1045 examples.put(args, description); 1046 1047 1048 args = new String[] 1049 { 1050 "--schemaDirectory", "/ds/config/schema", 1051 "--ldifFile", "data.ldif", 1052 "--rejectFile", "rejects.ldif", 1053 "--ignoreStructuralObjectClasses", 1054 "--ignoreAttributeSyntax" 1055 }; 1056 description = 1057 "Validate the contents of the 'data.ldif' file using the schema " + 1058 "defined in LDIF files contained in the /ds/config/schema directory " + 1059 "using a single thread. Any errors resulting from entries that do " + 1060 "not have exactly one structural object class or from values which " + 1061 "violate the syntax for their associated attribute types will be " + 1062 "ignored. Information about any other failures will be written to " + 1063 "the 'rejects.ldif' file."; 1064 examples.put(args, description); 1065 1066 return examples; 1067 } 1068 1069 1070 1071 /** 1072 * @return EntryValidator 1073 * 1074 * Returns the EntryValidator 1075 */ 1076 public EntryValidator getEntryValidator() 1077 { 1078 return entryValidator; 1079 } 1080}