U Ӈgn @sdZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z ddl m Z ddlmZddlmZddlmZdd lmZmZmZmZmZmZmZmZddlZdd lmZm Z m!Z!m"Z"dd l#m$Z$dd l%m&Z&m'Z'dd l(m)Z)ddl*m+Z+ddl,m-Z-ddl.m/Z/ddl0m1Z1m2Z2zddl3m4Z4Wne5k rRe6Z4YnXzddl7m8Z8m9Z9dZ:Wne5k rdZ:YnXe;e<Z=dZ>dZ?dZ@dZAeBdde&CDZDerddlZddlEmFZFmGZGGdddeGZHneIZHGdd d e4ZJGd!d"d"eZKeeKZLGd#d$d$eZMeMjNd%e>ieMjOd%e@ieMjPd%e?ieMjQd%e@iiZRGd&d'd'eZSGd(d)d)eZTGd*d+d+eZUdd,d-eLeeVeVeVd.d/d0ZWGd1d2d2eXZYGd3d4d4eYZZd5d6Z[dweIeVd7d8d9Z\ee4d:d;d<Z]eJfe^eIeVee6d=d>d?Z_ee_dd@Z`ee_dAd@ZaeJfee6dBdCdDZbeJfee6dBdEdFZcdGdHZddxeIdIdJdKZeeIeefdLdMdNZgdyeIe^e^e^e^dOdPdQZhe!idRdeMjNddddfeIeeIeMe^e^e^e^e^dSdTdUZjGdVdWdWZkdddXeVeIeeLeeLeeVelfdYdZd[ZmeVeeKd\d]d^ZndzeVeVeeVeUd_d`daZoeMjNddfeVeIeMe^eeVe^dbdcddZpeVd:dedfZqeMjNfeMeIdgdhdiZrd{djdkZsdldmZteeVeeTfd:dndoZudpdqZveVd:drdsZwdtduZxeGs r&) NotRequired TypedDictc@s<eZdZUeed<ejeed<eed<eeeed<dS) MetaSchemaidZdistrosZ frequencyZactivate_by_schema_keysN)__name__ __module__ __qualname__str__annotations__typingr r'r"r"r"r%r)Qs r)cs$eZdZeedfdd ZZS)SchemaDeprecationError)messageversionc stj|f|||_dSN)super__init__r3)selfr2r3kwargs __class__r"r%r6\szSchemaDeprecationError.__init__)r+r,r-r.r6 __classcell__r"r"r9r%r1[sr1c@s,eZdZUeed<eed<edddZdS) SchemaProblempathr2returncCs|jd|jS)N: )r=r2r7r"r"r%formatjszSchemaProblem.formatN)r+r,r-r.r/rBr"r"r"r%r<fs r<c@s eZdZdZdZdZdZdZdS) SchemaTypeaSupported schema types are either cloud-config or network-config. Vendordata and Vendordata2 format adheres to cloud-config schema type. Cloud Metadata is unique schema to each cloud platform and likely will not be represented in this enum. z cloud-confignetwork-configznetwork-config-v1znetwork-config-v2N)r+r,r-__doc__ CLOUD_CONFIGNETWORK_CONFIGNETWORK_CONFIG_V1NETWORK_CONFIG_V2r"r"r"r%rCqs rClatestc@s(eZdZdZdZdZdZdZddZdS) InstanceDataTypez-Types of instance data provided to cloud-initz user-datarDz vendor-dataz vendor2-datacCs|jSr4)valuerAr"r"r%__str__szInstanceDataType.__str__N) r+r,r-rEUSERDATArG VENDORDATA VENDOR2DATArMr"r"r"r%rKs rKc@s&eZdZUeed<eed<eed<dS)InstanceDataPart config_type schema_type config_pathN)r+r,r-rKr/rCr.r"r"r"r%rQs rQc@seZdZUeed<eed<dS)UserDataTypeAndDecodedContent userdata_typecontentN)r+r,r-r.r/r"r"r"r%rUs rU, prefix separator)schema_problemsrZr[r?cCs(|tdd|}|r$||}|S)NcSs|Sr4)rB)pr"r"r%z)_format_schema_problems..)joinmap)r\rZr[Z formattedr"r"r%_format_schema_problemss rbcs@eZdZdZd eeeedfdd ZedddZZ S) SchemaValidationErrorz.handle_problemszCloud config schema errors: ri"Cloud config schema deprecations: N)rerfr5r6)r7rerfror9rnr%r6s  zSchemaValidationError.__init__r>cCs t|jSr4)boolrerAr"r"r% has_errorssz SchemaValidationError.has_errors)NN) r+r,r-rEr SchemaProblemsr6rqrrr;r"r"r9r%rcs"rcc@seZdZdZdS)"SchemaValidationInvalidHeaderErrorz>Raised when no valid header is declared in the user-data file.N)r+r,r-rEr"r"r"r%rtsrtcCsBzddlm}Wntk r&YdSX|j|dp@t|tfS)zWTYPE_CHECKER override allowing bytes for string type For jsonschema v. 3.0.0+ r)Draft4ValidatorFstring) jsonschemaru ImportError TYPE_CHECKERis_type isinstancebytes)Zcheckerinstancerur"r"r%is_schema_byte_strings r~)configr?csRtdfdd }|r|ndddg}dt||}dd}||S) zcombine description with new/changed/deprecated message deprecated/changed/new keys require a _version key (this is verified in a unittest), a _description key is optional keycsr|sdS|dd}|dd|d}|d|d|}rbd|Sd |d S) NrgZ _descriptionZ_versionz z in version .  z **)get capitalizestrip)rZkey_descriptionvmsgannotaterr"r%format_messages   z:_add_deprecated_changed_or_new_msg..format_messager changednewrg description)r.r`rarrstrip)rr filter_keyrZ filter_keysZchanged_new_deprecatedrr"rr%"_add_deprecated_changed_or_new_msgs  rr>cCsg}d}t|tod|k}|D]}|r|d|jdididgkrV|gSt|dr~|jdddkr||q|jr|jddkr||qt|j|kr||qt|j|krt|j}|g}q|S)zReturn the best_match errors based on the deepest match in the json_path This is useful for anyOf and oneOf subschemas where the most-specific error tends to be the most appropriate. rtype propertiesenum json_pathN) r{dictschemarhasattrrappendr=len)errorsr}Z best_matchesZ path_depthrzerrr"r"r%cloud_init_deepest_matchess0      r)r rr error_typeccs,|r(t|d|gd}|||ddVdS)zJsonschema validator for `deprecated` items. It yields an instance of `error_type` if deprecated that must be handled, otherwise the instance is consider faulty. T)rrZdeprecated_versiondevelN)rr) _validatorr Z _instancerrrrr"r"r%r4s r)rrrc#sddlm}g}g}d}t|D]\} } t|j|| | d} ttfdd| } ttfdd| } | sz|| qt|trd|krd | d d krd |d| d krd }| EdH|| q |s||Vt d|f|dV|EdHdS)aJsonschema validator for `anyOf`. It treats occurrences of `error_type` as non-errors, but yield them for external processing. Useful to process schema annotations, as `deprecated`. Cloud-init's network schema under the `config` key has a complexity of allowing each list dict item to declare it's type with a `type` key which can contain the values: bond, bridge, nameserver, physical, route, vlan. This schema 'flexibility' makes it hard for the default jsonschema.exceptions.best_match function to find the correct schema failure because it typically returns the failing schema error based on the schema of greatest match depth. Since each anyOf dict matches the same depth into the network schema path, `best_match` just returns the first set of schema errors, which is almost always incorrect. To find a better schema match when encountering schema validation errors, cloud-init network schema introduced schema $defs with the prefix `anyOf_type_`. If the object we are validating contains a 'type' key, and one of the failing schema objects in an anyOf clause has a name of the format anyOf_type_XXX, raise those schema errors instead of calling best_match. r) best_matchFZ schema_pathcs t| Sr4r{err"r%r^sr_z_anyOf..cs t|Sr4rrrr"r%r^ur_rZ anyOf_typez$refrgZ anyOf_type_TNz.%r is not valid under any of the given schemas)context) jsonschema.exceptionsr enumeraterkdescendfilterextendr{rrr) validatoranyOfr}_schemarr all_errorsall_deprecationsZskip_best_matchindex subschemaall_errserrs deprecationsr"rr%_anyOfLs@     rc#st|}g}g}|D]h\}} tj| |d} ttfdd| } ttfdd| } | sr| } || q|| qt|EdHfdd|D}|r|| dd d |D}td |fVn |EdHdS) zJsonschema validator for `oneOf`. It treats occurrences of `error_type` as non-errors, but yield them for external processing. Useful to process schema annotations, as `deprecated`. rcs t| Sr4rrrr"r%r^r_z_oneOf..cs t|Sr4rrrr"r%r^r_Ncs g|]\}}|r|qSr")is_valid)r#is)r}rr"r%r&s z_oneOf..rXcss|]}t|VqdSr4)repr)r#rr"r"r% sz_oneOf..z%r is valid under each of %s) rrkrrrrrr`r)roneOfr}rrZ subschemasrrrrrrrZ first_validZ more_validZreprsr")rr}rr%_oneOfs2      rc Csddlm}m}ddlm}t|j}ddi|dd<i}d|d <|jdt }d |i}t |j }t |t <t|d <t|d <t|d <|f||dd|}ddd}ddd} |} t|dr| } | |_||fS)zGet metaschema validator and format checker Older versions of jsonschema require some compatibility changes. @returns: Tuple: (jsonschema.Validator, FormatChecker) @raises: ImportError when jsonschema is not present r)ru FormatChecker)createrrvrlabelFadditionalProperties type_checkerrrrZdraft4) meta_schema validatorsr3Nc[s$tdd|||}t|ddkS)gOverride version of `is_valid`. It does ignore instances of `SchemaDeprecationError`. cSs t|t Sr4r{r1rr"r"r%r^szFget_jsonschema_validator..is_valid_pre_4_0_0..N)r iter_errorsnextr7r}r__rr"r"r%is_valid_pre_4_0_0s  z4get_jsonschema_validator..is_valid_pre_4_0_0c[s*tdd|j|d|}t|ddkS)rcSs t|t Sr4rrr"r"r%r^sz.is_valid..rN)revolverrrr"r"r%rs z*get_jsonschema_validator..is_validr)N)N)rwrurZjsonschema.validatorsrrZ META_SCHEMAryZredefiner~rZ VALIDATORS_validator_deprecatedDEPRECATED_KEY_validator_changedrrrr) rurrrZvalidator_kwargsrrcloudinitValidatorrrZ is_valid_fnr"r"r%get_jsonschema_validators@     rrc Cszddlm}z||WnZ|k rt}z<|rXttddd|jD|jgd|t d|W5d}~XYnXdS) a Validate provided schema meets the metaschema definition. Return strict Validator and FormatChecker for use in validation @param validator: Draft4Validator instance used to validate the schema @param schema: schema to validate @param throw: Sometimes the validator and checker are required, even if the schema is invalid. Toggle for whether to raise SchemaValidationError or log warnings. @raises: ImportError when jsonschema is not present @raises: SchemaValidationError when the schema is invalid r) SchemaError.cSsg|] }t|qSr"r.r#r]r"r"r%r&'sz3validate_cloudconfig_metaschema..rezGMeta-schema validation failed, attempting to validate config anyway: %sN) rrZ check_schemarcr<r`r=r2LOGwarning)rrthrowrrr"r"r%validate_cloudconfig_metaschemas$ r)network_configr?cCs d|kr|ddS|dS)z6Return the version of the network schema when present.networkr3)r)rr"r"r%network_schema_version2sr)rstrictr log_detailsr?c CsPtrtdntddSt}tj|d}t|}d|krJd|i}t |}t ||ddt }g} z| |WnHt k r} z*| tdj| j| jd d | jW5d } ~ XYnXtj|rt|| rL|r|r t|\} } tt|| | d t| |r4t| tjjd dd} ntjjd} t| dS)ajOn systems with netplan, validate network_config schema for file Leverage NetplanParser for error annotation line, column and detailed errors. @param network_config: Dict of network configuration settings validated against @param strict: Boolean, when True raise SchemaValidationErrors instead of logging warnings. @param annotate: Boolean, when True, print original network_config_file content with error annotations @param log_details: Boolean, when True logs details of validation errors. If there are concerns about logging sensitive userdata, this should be set to False. @return: True when schema validation was performed. False when not on a system with netplan and netplan python support. @raises: SchemaValidationError when netplan's parser raises NetplanParserExceptions. z*Validating network-config with netplan APIz.rrcSsg|] }t|qSr"rrr"r"r%r&sz/validate_cloudconfig_schema..rz#.*\('(?P.*)' was unexpected\)namerz"Deprecated cloud-config provided: rirrrYr)&cloudinit.net.netplanrrCrGrrIrH get_schemarrrrxrrrjrr`r=rrrematchr2r{r1r3rZshould_log_deprecationrZDEPRECATION_INFO_BOUNDARYrr<rbinfor rcrLr)rrrSrrrrnetplan_availablenetwork_versionrrrrrZinfo_deprecations schema_errorr=Z prop_matchr2Zdetailsr"r"r%validate_cloudconfig_schemas$             r c @seZdZeedddZeeeeedddZe ddd Z edeeeeeee ee d d d Z eeeeeedddZ e e edddZdS) _Annotator)original_content schemamarkscCs||_||_dSr4)_original_content _schemamarks)r7r r r"r"r%r6)sz_Annotator.__init__)titlerWr?cCsd|}d|d|dS)Nr# z: ------------- rh)r`)rrWZbodyr"r"r% _build_footer1s z_Annotator._build_footer)r\cCsztt}|D]h\}}td|}|rD|\}}|t||nd}||j|||dk r dj|||d}q |S)Nz&format-l(?P\d+)\.c(?P\d+).*zLine {line} column {col}: {msg})rrr) rrkrrgroupsintrrrB)r7r\errors_by_liner=rrrrr"r"r%_build_errors_by_line6s   z _Annotator._build_errors_by_linerg)rmlabelsfooterr label_prefixr?cCsB|D]8}||}|||d|d||d7}q|S)Nrr@r)r)rmrrrrZproblemrr"r"r% _add_problemsFs    z_Annotator._add_problems)linesrdeprecations_by_liner?c sg}g}g}d}d}t|dD]p\} } || } || } | s>| rg} j| | ||dd}j| | ||dd}|| dd| q|| q|tfddtd dd |fd |ff|S) NrE)rDz # ,cs j|Sr4)rseqrAr"r%r^ur_z._Annotator._annotate_content..cSs t|dS)Nr)rqrr"r"r%r^wr_ZErrorsZ Deprecations)rrrr`rrar)r7rrrannotated_contentZ error_footerZdeprecation_footerZ error_indexZdeprecation_indexZ line_numberrrrrr"rAr%_annotate_contentUsL   z_Annotator._annotate_content)rerfr?cCsF|s|s|jS|jd}||}||}||||}d|S)Nr)r splitrr"r`)r7rerfrrrr!r"r"r%rs   z_Annotator.annotateN)rg)r+r,r-r.rr6 staticmethodr rrsrrrr"rr"r"r"r%r (s2  .r rd)r r rerfr?cCst|||pg|pgS)aReturn contents of the cloud-config file annotated with schema errors. @param cloudconfig: YAML-loaded dict from the original_content or empty dict if unparsable. @param original_content: The contents of a cloud-config file @param schemamarks: Dict with schema marks. @param schema_errors: Instance of `SchemaProblems`. @param schema_deprecations: Instance of `SchemaProblems`. @return Annotated schema )r r)r r rerfr"r"r%rs r)rWr?c Csbddlm}||krgSg}t|dD]2\}}||r*|td|d||dq*|S)aAnnotate and return schema validation errors in merged cloud-config.txt When merging multiple cloud-config parts cloud-init logs an error and ignores any user-data parts which are declared as #cloud-config but cannot be processed. the handler.cloud_config module also leaves comments in the final merged config for every invalid part file which begin with MERGED_CONFIG_SCHEMA_ERROR_PREFIX to aid in triage. r)MERGED_PART_SCHEMA_ERROR_PREFIXrzformat-lz.c1zIgnored invalid user-data: )Zcloudinit.handlers.cloud_configr%r splitlines startswithrr<replace)rWr%rZline_numrr"r"r%)process_merged_cloud_config_part_problemss    r))rTrWinstance_data_pathr?c CsFddlm}m}m}m}t|}d}|dkrz||||}Wn|k rp} ztt|dg| W5d} ~ XYnf|k r} ztdt | dd W5d} ~ XYn2|k r} ztt | dd W5d} ~ XYnXd }t|}|s"| d \} } } t t|d |d | dd t gn|dkr.rz3Expected one of --config-file or --system argumentsTr/zMWARNING: The --schema-type parameter is inapplicable when --system is presentN) config_filesystemrrrSr)argsZexclusive_argsr"r"r%_assert_exclusive_argss  rUc Csttttddd}ztdd}Wnfttfk rf}z"|jtkrTtdt}nW5d}~XYn$t k rt}t dYnX|j r|j }n"t d kr|d }n |d }g}|jr |jrt|j}ntj}|tjkrtj}ntj}|t|||jnt d kr&td d d||dd}|ttjtj|ttjtj||ddttjtj||ddttjtj|dpdg} | D](} | jrt j| jr|| qt j|d jstd|d jddd d||fS)aReturn appropriate instance-data.json and instance data parts Based on command line args, and user permissions, determine the appropriate instance-data.json to source for jinja templates and a list of applicable InstanceDataParts such as user-data, vendor-data and network-config for which to validate schema. Avoid returning any InstanceDataParts when the expected config_path does not exist. :return: A tuple of the instance-data.json path and a list of viable InstanceDataParts present on the system. )pathsprimary_path_keyraw_fallback_path_keyr?c Ss\||p d}tt<t|jsN||p0d}t|jrN|W5QRSW5QRX|S)akGet processed data path when non-empty of fallback to raw data path. - When primary path and raw path exist and are empty, prefer primary path. - When primary path is empty but the raw fallback path is non-empty, this indicates an invalid and ignored raw user-data was provided and cloud-init emitted a warning and did not process unknown raw user-data. In the case of invalid raw user-data header, prefer raw_fallback_path_key so actionable sensible warnings can be reported to the user about the raw unparsable user-data. rg) get_ipathrFileNotFoundErrorrstatst_size)rVrWrXZprimary_datapathZraw_pathr"r"r%get_processed_or_fallback_path s   zBget_config_paths_from_args..get_processed_or_fallback_pathZtrust)Zfetch_existing_datasourcez=Using default instance-data/user-data paths for non-root userNzEdatasource not detected, using default instance-data/user-data paths.rrGZinstance_data_sensitivezNUnable to read system userdata or vendordata as non-root user. Try using sudo.Tr/Z cloud_configZ userdata_rawZvendor_cloud_configZvendordata_rawZvendor2_cloud_configZvendordata2_rawrrgz Config file z does not existz Error: {}fmtr0)rr.rr@rAerrnorrrrrrGrgetuidrNrRrSrCrFrGrKrNrrQrrOrPrYrTr=r) rTr]rVrr* config_filesrSZinstancedata_typeZ userdata_fileZsupplemental_config_filesZ data_partr"r"r%get_config_paths_from_argss         rcc Cst|t}t|\}}d}tt|dk}|rRtdddd|Dd}g}t|dD]V\}} d} |rtd |d | jd | j d | j t j krt| j } n|} zt | j | | j |j|} Wntk r0} zH|jst|d | jd| j tt| |dd|| jW5d} ~ XYq`tk r} z8t|d | jtt| |dd|| jW5d} ~ XYq`X| r`|jr| j } n t| j} t|d| q`|rtddd|DddddS)z@Handle provided schema args and perform the appropriate actions.rgrz!Found cloud-config data types: %srXcss|]}t|jVqdSr4)r.rR)r#cfg_partr"r"r%rsz%handle_schema_args..z Frrz at :zInvalid rz Error: {} )r_Nz Valid schema css|]}t|VqdSr4r)r#rr"r"r%rszError: Invalid schema: {} Tr^)rUrrcrqrrr`rrRrTrSrCrGr8rrcrr.rr7rR)rrTrBr*rbZnested_output_prefixZmulti_config_outputZ error_typesidxrdZperformed_schema_validationZ cfg_schemarZcfgr"r"r%handle_schema_argsxsn    rgcOstjtdtjddddS)zProvide a stub for backwards compatibility. This function is no longer used, but earlier versions of modules required this function for documentation purposes. This is a stub so that custom modules do not break on upgrade. z24.4zbThe 'get_meta_doc()' function is deprecated and will be removed in a future version of cloud-init.r")Zloggerr3Zrequested_levelrrTrg)rZlog_with_downgradable_levelrloggingZWARNING)Z_argsZ_kwargsr"r"r% get_meta_docs ricCst}td|dS)zDTool to validate schema of a cloud-config file or print schema docs.rCr)rPrg parse_argsrOr"r"r%mainsrk__main__)FN)T)FFT)N)N)zrErKr>rhrrrsys collectionsr contextlibrcopyrrrr`r functoolsrr0rr r r r r rrr5Z cloudinitrrrrZcloudinit.cmd.develrZcloudinit.handlersrrZcloudinit.helpersrZcloudinit.log.log_utilrZcloudinit.sourcesrZcloudinit.temp_utilsrZcloudinit.utilrrrwrrx ExceptionZnetplanrrrZ getLoggerr+rZUSERDATA_SCHEMA_FILEZNETWORK_CONFIG_V1_SCHEMA_FILEZNETWORK_CONFIG_V2_SCHEMA_FILErrjkeysr3Ztyping_extensionsr'r(r)rr1r<rsrCrFrGrHrIr=rKrQrUr.rb ValueErrorrcrtr~rrrqrrrrrrrrrrZtimedr r r|rr)r4r8r<rrPrUrcrgrirkexitr"r"r"r%sd      (            ) #'    H *V#  _m  ! H  ; }@