From a365abdb0c24606de4f53f354ba830ec9ee07ca6 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Fri, 16 Nov 2018 15:39:19 +0000 Subject: [PATCH 1/9] Prep to merge from google/master --- validator/build.py | 397 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 304 insertions(+), 93 deletions(-) diff --git a/validator/build.py b/validator/build.py index ce0a78c67b4e..b87ec9ea2edb 100755 --- a/validator/build.py +++ b/validator/build.py @@ -16,6 +16,7 @@ # """A build script which (thus far) works on Ubuntu 14.""" +import argparse import glob import logging import os @@ -37,19 +38,17 @@ def Die(msg): sys.exit(1) -def GetNodeJsCmd(): - """Ensure Node.js is installed and return the proper command to run.""" +def EnsureNodeJsIsInstalled(): + """Ensure Node.js is installed and that 'node' is the command to run.""" logging.info('entering ...') - for cmd in ['node', 'nodejs']: try: - output = subprocess.check_output([cmd, '--eval', 'console.log("42")']) + output = subprocess.check_output(['node', '--eval', 'console.log("42")']) if output.strip() == '42': - logging.info('... done') - return cmd + return except (subprocess.CalledProcessError, OSError): - continue - Die('Node.js not found. Try "apt-get install nodejs".') + pass + Die('Node.js not found. Try "apt-get install nodejs" or follow the install instructions at https://github.com/ampproject/amphtml/blob/master/validator/README.md#installation') def CheckPrereqs(): @@ -61,10 +60,13 @@ def CheckPrereqs(): 'Please feel free to edit the source and fix it to your needs.') # Ensure source files are available. - for f in ['validator-main.protoascii', 'validator.proto', - 'validator_gen_js.py', 'package.json', 'validator.js', - 'validator_test.js', 'validator-in-browser.js', 'tokenize-css.js', - 'parse-css.js', 'parse-srcset.js']: + for f in [ + 'validator-main.protoascii', 'validator.proto', 'validator_gen_js.py', + 'package.json', 'engine/validator.js', 'engine/validator_test.js', + 'engine/validator-in-browser.js', 'engine/tokenize-css.js', + 'engine/definitions.js', 'engine/parse-css.js', 'engine/parse-srcset.js', + 'engine/parse-url.js' + ]: if not os.path.exists(f): Die('%s not found. Must run in amp_validator source directory.' % f) @@ -72,7 +74,7 @@ def CheckPrereqs(): try: libprotoc_version = subprocess.check_output(['protoc', '--version']) except (subprocess.CalledProcessError, OSError): - Die('Protobuf compiler not found. Try "apt-get install protobuf-compiler".') + Die('Protobuf compiler not found. Try "apt-get install protobuf-compiler" or follow the install instructions at https://github.com/ampproject/amphtml/blob/master/validator/README.md#installation.') # Ensure 'libprotoc 2.5.0' or newer. m = re.search('^(\\w+) (\\d+)\\.(\\d+)\\.(\\d+)', libprotoc_version) @@ -86,24 +88,21 @@ def CheckPrereqs(): try: __import__(module) except ImportError: - Die('%s not found. Try "apt-get install python-protobuf"' % module) + Die('%s not found. Try "apt-get install python-protobuf" or follow the install instructions at https://github.com/ampproject/amphtml/blob/master/validator/README.md#installation' % module) - # Ensure that npm is installed. + # Ensure that yarn is installed. try: - npm_version = subprocess.check_output(['npm', '--version']) + subprocess.check_output(['yarn', '--version']) except (subprocess.CalledProcessError, OSError): - Die('npm package manager not found. Try "apt-get install npm".') - - # Ensure npm version '1.3.10' or newer. - m = re.search('^(\\d+)\\.(\\d+)\\.(\\d+)$', npm_version) - if (int(m.group(1)), int(m.group(2)), int(m.group(3))) < (1, 3, 10): - Die('Expected npm version 1.3.10 or newer, saw: %s' % npm_version) + Die('Yarn package manager not found. Run ' + '"curl -o- -L https://yarnpkg.com/install.sh | bash" ' + 'or see https://yarnpkg.com/docs/install.') # Ensure JVM installed. TODO: Check for version? try: subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT) except (subprocess.CalledProcessError, OSError): - Die('Java missing. Try "apt-get install openjdk-7-jre"') + Die('Java missing. Try "apt-get install openjdk-7-jre" or follow the install instructions at https://github.com/ampproject/amphtml/blob/master/validator/README.md#installation') logging.info('... done') @@ -124,10 +123,19 @@ def SetupOutDir(out_dir): def InstallNodeDependencies(): + """Installs the dependencies using yarn.""" logging.info('entering ...') # Install the project dependencies specified in package.json into # node_modules. - subprocess.check_call(['npm', 'install']) + logging.info('installing AMP Validator engine dependencies ...') + subprocess.check_call( + ['yarn', 'install'], + stdout=(open(os.devnull, 'wb') if os.environ.get('TRAVIS') else sys.stdout)) + logging.info('installing AMP Validator nodejs dependencies ...') + subprocess.check_call( + ['yarn', 'install'], + cwd='nodejs', + stdout=(open(os.devnull, 'wb') if os.environ.get('TRAVIS') else sys.stdout)) logging.info('... done') @@ -141,8 +149,8 @@ def GenValidatorPb2Py(out_dir): logging.info('entering ...') assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir - subprocess.check_call(['protoc', 'validator.proto', '--python_out=%s' % - out_dir]) + subprocess.check_call( + ['protoc', 'validator.proto', '--python_out=%s' % out_dir]) open('%s/__init__.py' % out_dir, 'w').close() logging.info('... done') @@ -158,11 +166,11 @@ def GenValidatorProtoascii(out_dir): assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir protoascii_segments = [open('validator-main.protoascii').read()] - extensions = glob.glob('extensions/*/0.1/validator-*.protoascii') + extensions = glob.glob('extensions/*/validator-*.protoascii') # In the Github project, the extensions are located in a sibling directory # to the validator rather than a child directory. if not extensions: - extensions = glob.glob('../extensions/*/0.1/validator-*.protoascii') + extensions = glob.glob('../extensions/*/validator-*.protoascii') extensions.sort() for extension in extensions: protoascii_segments.append(open(extension).read()) @@ -172,6 +180,42 @@ def GenValidatorProtoascii(out_dir): logging.info('... done') +def GenValidatorProtoGeneratedJs(out_dir): + """Calls validator_gen_js to generate validator-proto-generated.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ + logging.info('entering ...') + assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir + + # These imports happen late, within this method because they don't necessarily + # exist when the module starts running, and the ones that probably do + # are checked by CheckPrereqs. + # pylint: disable=g-import-not-at-top + from google.protobuf import text_format + from google.protobuf import descriptor + from dist import validator_pb2 + import validator_gen_js + # pylint: enable=g-import-not-at-top + out = [] + validator_gen_js.GenerateValidatorGeneratedJs( + specfile=None, + validator_pb2=validator_pb2, + generate_proto_only=True, + generate_spec_only=False, + text_format=text_format, + html_format=None, + descriptor=descriptor, + out=out) + out.append('') + f = open('%s/validator-proto-generated.js' % out_dir, 'w') + f.write('\n'.join(out)) + f.close() + logging.info('... done') + + def GenValidatorGeneratedJs(out_dir): """Calls validator_gen_js to generate validator-generated.js. @@ -185,15 +229,20 @@ def GenValidatorGeneratedJs(out_dir): # These imports happen late, within this method because they don't necessarily # exist when the module starts running, and the ones that probably do # are checked by CheckPrereqs. + # pylint: disable=g-import-not-at-top from google.protobuf import text_format from google.protobuf import descriptor from dist import validator_pb2 import validator_gen_js + # pylint: enable=g-import-not-at-top out = [] validator_gen_js.GenerateValidatorGeneratedJs( specfile='%s/validator.protoascii' % out_dir, validator_pb2=validator_pb2, + generate_proto_only=False, + generate_spec_only=True, text_format=text_format, + html_format=None, descriptor=descriptor, out=out) out.append('') @@ -261,24 +310,31 @@ def GenValidatorGeneratedMd(out_dir): f.close() logging.info('... done') -def CompileWithClosure(js_files, closure_entry_points, output_file): +def CompileWithClosure(js_files, definitions, entry_points, output_file): """Compiles the arguments with the Closure compiler for transpilation to ES5. Args: js_files: list of files to compile - closure_entry_points: entry points (these won't be minimized) + definitions: list of definitions flags to closure compiler + entry_points: entry points (these won't be minimized) output_file: name of the Javascript output file """ - cmd = ['java', '-jar', 'node_modules/google-closure-compiler/compiler.jar', - '--language_in=ECMASCRIPT6_STRICT', '--language_out=ES5_STRICT', - '--js_output_file=%s' % output_file, '--only_closure_dependencies'] - cmd += ['--closure_entry_point=%s' % e for e in closure_entry_points] - cmd += ['node_modules/google-closure-library/closure/**.js', + cmd = [ + 'java', '-jar', 'node_modules/google-closure-compiler/compiler.jar', + '--language_out=ES5_STRICT', '--dependency_mode=STRICT', + '--js_output_file=%s' % output_file + ] + cmd += ['--entry_point=%s' % e for e in entry_points] + cmd += ['--output_manifest=%s' % ('%s.manifest' % output_file)] + cmd += [ + 'node_modules/google-closure-library/closure/**.js', '!node_modules/google-closure-library/closure/**_test.js', 'node_modules/google-closure-library/third_party/closure/**.js', - '!node_modules/google-closure-library/third_party/closure/**_test.js'] + '!node_modules/google-closure-library/third_party/closure/**_test.js' + ] cmd += js_files + cmd += definitions subprocess.check_call(cmd) @@ -290,43 +346,55 @@ def CompileValidatorMinified(out_dir): """ logging.info('entering ...') CompileWithClosure( - js_files=['htmlparser.js', 'parse-css.js', 'parse-srcset.js', - 'tokenize-css.js', '%s/validator-generated.js' % out_dir, - 'validator-in-browser.js', 'validator.js', 'validator-full.js', - 'htmlparser-interface.js'], - closure_entry_points=['amp.validator.validateString', - 'amp.validator.renderValidationResult', - 'amp.validator.renderErrorMessage'], + js_files=[ + 'engine/definitions.js', 'engine/htmlparser.js', + 'engine/parse-css.js', 'engine/parse-srcset.js', + 'engine/parse-url.js', 'engine/tokenize-css.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir, + 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/validator-full.js', + 'engine/amp4ads-parse-css.js', 'engine/keyframes-parse-css.js', + 'engine/htmlparser-interface.js' + ], + definitions=[], + entry_points=[ + 'amp.validator.validateString', + 'amp.validator.renderValidationResult', + 'amp.validator.renderErrorMessage' + ], output_file='%s/validator_minified.js' % out_dir) logging.info('... done') -def RunSmokeTest(out_dir, nodejs_cmd): +def RunSmokeTest(out_dir): """Runs a smoke test (minimum valid AMP and empty html file). Args: out_dir: output directory - nodejs_cmd: the command for calling Node.js """ logging.info('entering ...') # Run index.js on the minimum valid amp and observe that it passes. p = subprocess.Popen( - [nodejs_cmd, 'index.js', '--validator_js', + [ + 'node', 'nodejs/index.js', '--validator_js', '%s/validator_minified.js' % out_dir, - 'testdata/feature_tests/minimum_valid_amp.html'], + 'testdata/feature_tests/minimum_valid_amp.html', '--format=text' + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() - if ('testdata/feature_tests/minimum_valid_amp.html: PASS\n', '', - p.returncode) != (stdout, stderr, 0): + if ('testdata/feature_tests/minimum_valid_amp.html: PASS\n', '', p.returncode + ) != (stdout, stderr, 0): Die('Smoke test failed. returncode=%d stdout="%s" stderr="%s"' % (p.returncode, stdout, stderr)) # Run index.js on an empty file and observe that it fails. p = subprocess.Popen( - [nodejs_cmd, 'index.js', '--validator_js', + [ + 'node', 'nodejs/index.js', '--validator_js', '%s/validator_minified.js' % out_dir, - 'testdata/feature_tests/empty.html'], + 'testdata/feature_tests/empty.html', '--format=text' + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() @@ -338,16 +406,15 @@ def RunSmokeTest(out_dir, nodejs_cmd): logging.info('... done') -def RunIndexTest(nodejs_cmd): +def RunIndexTest(): """Runs the index_test.js, which tests the NodeJS API. - - Args: - nodejs_cmd: the command for calling Node.js """ logging.info('entering ...') - p = subprocess.Popen([nodejs_cmd, 'index_test.js'], + p = subprocess.Popen( + ['node', './index_test.js'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.PIPE, + cwd='nodejs') (stdout, stderr) = p.communicate() if p.returncode != 0: Die('index_test.js failed. returncode=%d stdout="%s" stderr="%s"' % @@ -356,13 +423,26 @@ def RunIndexTest(nodejs_cmd): def CompileValidatorTestMinified(out_dir): + """Runs closure compiler for validator_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ logging.info('entering ...') CompileWithClosure( - js_files=['htmlparser.js', 'parse-css.js', 'parse-srcset.js', - 'tokenize-css.js', '%s/validator-generated.js' % out_dir, - 'validator-in-browser.js', 'validator.js', 'validator-full.js', - 'htmlparser-interface.js', 'validator_test.js'], - closure_entry_points=['amp.validator.ValidatorTest'], + js_files=[ + 'engine/definitions.js', 'engine/htmlparser.js', + 'engine/parse-css.js', 'engine/parse-srcset.js', + 'engine/parse-url.js', 'engine/tokenize-css.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir, + 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/validator-full.js', + 'engine/amp4ads-parse-css.js', 'engine/keyframes-parse-css.js', + 'engine/htmlparser-interface.js', 'engine/validator_test.js' + ], + definitions=[], + entry_points=['amp.validator.ValidatorTest'], output_file='%s/validator_test_minified.js' % out_dir) logging.info('... success') @@ -370,48 +450,155 @@ def CompileValidatorTestMinified(out_dir): def CompileValidatorLightTestMinified(out_dir): logging.info('entering ...') CompileWithClosure( - js_files=['htmlparser.js', 'parse-css.js', 'parse-srcset.js', - 'tokenize-css.js', '%s/validator-generated.js' % out_dir, - 'validator-in-browser.js', 'validator.js', 'validator-light.js', - 'htmlparser-interface.js', 'dom-walker.js', - 'validator-light_test.js'], - closure_entry_points=['amp.validator.ValidatorTest'], + js_files=['engine/htmlparser.js', 'engine/parse-css.js', 'engine/parse-srcset.js', + 'engine/tokenize-css.js', '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir, + 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/validator-light.js', + 'engine/htmlparser-interface.js', 'engine/dom-walker.js', + 'engine/validator-light_test.js'], + definitions=[], + entry_points=['amp.validator.ValidatorTest'], output_file='%s/validator-light_test_minified.js' % out_dir) logging.info('... success') def CompileHtmlparserTestMinified(out_dir): + """Runs closure compiler for htmlparser_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ logging.info('entering ...') - CompileWithClosure(js_files=['htmlparser.js', 'htmlparser-interface.js', - 'htmlparser_test.js'], - closure_entry_points=['amp.htmlparser.HtmlParserTest'], + CompileWithClosure( + js_files=[ + 'engine/htmlparser.js', 'engine/htmlparser-interface.js', + 'engine/htmlparser_test.js' + ], + definitions=[], + entry_points=['amp.htmlparser.HtmlParserTest'], output_file='%s/htmlparser_test_minified.js' % out_dir) logging.info('... success') def CompileParseCssTestMinified(out_dir): + """Runs closure compiler for parse-css_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ logging.info('entering ...') CompileWithClosure( - js_files=['parse-css.js', 'tokenize-css.js', 'css-selectors.js', - 'json-testutil.js', 'parse-css_test.js', - '%s/validator-generated.js' % out_dir], - closure_entry_points=['parse_css.ParseCssTest'], + js_files=[ + 'engine/definitions.js', 'engine/parse-css.js', 'engine/parse-url.js', + 'engine/tokenize-css.js', 'engine/css-selectors.js', + 'engine/json-testutil.js', 'engine/parse-css_test.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir + ], + definitions=[], + entry_points=['parse_css.ParseCssTest'], output_file='%s/parse-css_test_minified.js' % out_dir) logging.info('... success') +def CompileParseUrlTestMinified(out_dir): + """Runs closure compiler for parse-url_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ + logging.info('entering ...') + CompileWithClosure( + js_files=[ + 'engine/definitions.js', 'engine/parse-url.js', 'engine/parse-css.js', + 'engine/tokenize-css.js', 'engine/css-selectors.js', + 'engine/json-testutil.js', 'engine/parse-url_test.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir + ], + definitions=[], + entry_points=['parse_url.ParseURLTest'], + output_file='%s/parse-url_test_minified.js' % out_dir) + logging.info('... success') + + +def CompileAmp4AdsParseCssTestMinified(out_dir): + """Runs closure compiler for amp4ads-parse-css_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ + logging.info('entering ...') + CompileWithClosure( + js_files=[ + 'engine/definitions.js', 'engine/amp4ads-parse-css_test.js', + 'engine/parse-css.js', 'engine/parse-url.js', + 'engine/amp4ads-parse-css.js', 'engine/tokenize-css.js', + 'engine/css-selectors.js', 'engine/json-testutil.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir + ], + definitions=[], + entry_points=['parse_css.Amp4AdsParseCssTest'], + output_file='%s/amp4ads-parse-css_test_minified.js' % out_dir) + logging.info('... success') + + +def CompileKeyframesParseCssTestMinified(out_dir): + """Runs closure compiler for keyframes-parse-css_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ + logging.info('entering ...') + CompileWithClosure( + js_files=[ + 'engine/definitions.js', 'engine/keyframes-parse-css_test.js', + 'engine/parse-css.js', 'engine/parse-url.js', + 'engine/keyframes-parse-css.js', 'engine/tokenize-css.js', + 'engine/css-selectors.js', 'engine/json-testutil.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir + ], + definitions=[], + entry_points=['parse_css.KeyframesParseCssTest'], + output_file='%s/keyframes-parse-css_test_minified.js' % out_dir) + logging.info('... success') + + def CompileParseSrcsetTestMinified(out_dir): + """Runs closure compiler for parse-srcset_test.js. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ logging.info('entering ...') CompileWithClosure( - js_files=['parse-srcset.js', 'json-testutil.js', 'parse-srcset_test.js', - '%s/validator-generated.js' % out_dir], - closure_entry_points=['parse_srcset.ParseSrcsetTest'], + js_files=[ + 'engine/definitions.js', 'engine/parse-srcset.js', + 'engine/json-testutil.js', 'engine/parse-srcset_test.js', + '%s/validator-generated.js' % out_dir, + '%s/validator-proto-generated.js' % out_dir + ], + definitions=[], + entry_points=['parse_srcset.ParseSrcsetTest'], output_file='%s/parse-srcset_test_minified.js' % out_dir) logging.info('... success') def GenerateTestRunner(out_dir): - """Generates a test runner: a nodejs script that runs our minified tests.""" + """Generates a test runner: a nodejs script that runs our minified tests. + + Args: + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ logging.info('entering ...') f = open('%s/test_runner' % out_dir, 'w') extensions_dir = 'extensions' @@ -419,7 +606,7 @@ def GenerateTestRunner(out_dir): # to the validator rather than a child directory. if not os.path.isdir(extensions_dir): extensions_dir = '../extensions' - f.write("""#!/usr/bin/nodejs + f.write("""#!/usr/bin/env node global.assert = require('assert'); global.fs = require('fs'); global.path = require('path'); @@ -430,6 +617,9 @@ def GenerateTestRunner(out_dir): require('./validator-light_test_minified'); require('./htmlparser_test_minified'); require('./parse-css_test_minified'); + require('./parse-url_test_minified'); + require('./amp4ads-parse-css_test_minified'); + require('./keyframes-parse-css_test_minified'); require('./parse-srcset_test_minified'); jasmine.onComplete(function (passed) { process.exit(passed ? 0 : 1); @@ -440,12 +630,22 @@ def GenerateTestRunner(out_dir): logging.info('... success') -def RunTests(out_dir, nodejs_cmd): +def RunTests(update_tests, out_dir): + """Runs all the minified tests. + + Args: + update_tests: a boolean indicating whether or not to update the test + output files. + out_dir: directory name of the output directory. Must not have slashes, + dots, etc. + """ logging.info('entering ...') - subprocess.check_call([nodejs_cmd, '%s/test_runner' % out_dir]) + env = os.environ.copy() + if update_tests: + env['UPDATE_VALIDATOR_TEST'] = '1' + subprocess.check_call(['node', '%s/test_runner' % out_dir], env=env) logging.info('... success') - def CreateWebuiAppengineDist(out_dir): logging.info('entering ...') try: @@ -472,31 +672,42 @@ def CreateWebuiAppengineDist(out_dir): logging.info('... success') -def Main(): +def Main(parsed_args): """The main method, which executes all build steps and runs the tests.""" - logging.basicConfig(format='[[%(filename)s %(funcName)s]] - %(message)s', - level=logging.INFO) - nodejs_cmd = GetNodeJsCmd() + logging.basicConfig( + format='[[%(filename)s %(funcName)s]] - %(message)s', + level=(logging.ERROR if os.environ.get('TRAVIS') else logging.INFO)) + EnsureNodeJsIsInstalled() CheckPrereqs() InstallNodeDependencies() SetupOutDir(out_dir='dist') GenValidatorProtoascii(out_dir='dist') GenValidatorPb2Py(out_dir='dist') - GenValidatorProtoascii(out_dir='dist') - GenValidatorGeneratedJs(out_dir='dist') GenValidatorGeneratedPHP(out_dir='dist') GenValidatorGeneratedMd(out_dir='dist') + GenValidatorProtoGeneratedJs(out_dir='dist') + GenValidatorGeneratedJs(out_dir='dist') CompileValidatorMinified(out_dir='dist') - RunSmokeTest(out_dir='dist', nodejs_cmd=nodejs_cmd) - RunIndexTest(nodejs_cmd=nodejs_cmd) + RunSmokeTest(out_dir='dist') + RunIndexTest() CompileValidatorTestMinified(out_dir='dist') CompileValidatorLightTestMinified(out_dir='dist') CompileHtmlparserTestMinified(out_dir='dist') CompileParseCssTestMinified(out_dir='dist') + CompileParseUrlTestMinified(out_dir='dist') + CompileAmp4AdsParseCssTestMinified(out_dir='dist') + CompileKeyframesParseCssTestMinified(out_dir='dist') CompileParseSrcsetTestMinified(out_dir='dist') GenerateTestRunner(out_dir='dist') - RunTests(out_dir='dist', nodejs_cmd=nodejs_cmd) + RunTests(update_tests=parsed_args.update_tests, out_dir='dist') CreateWebuiAppengineDist(out_dir='dist') if __name__ == '__main__': - Main() + parser = argparse.ArgumentParser( + description='Build script for the AMP Validator.') + parser.add_argument( + '--update_tests', + action='store_true', + help=('If True, validator_test will overwrite the .out test files with ' + 'the encountered test output.')) + Main(parser.parse_args()) From d4d17ec9f2746d5ec4402f47186776368b8e28d9 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Fri, 16 Nov 2018 16:28:45 +0000 Subject: [PATCH 2/9] Indentation --- validator/build.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/validator/build.py b/validator/build.py index 8285586ce320..54270ad3d295 100755 --- a/validator/build.py +++ b/validator/build.py @@ -42,15 +42,13 @@ def EnsureNodeJsIsInstalled(): """Ensure Node.js is installed and that 'node' is the command to run.""" logging.info('entering ...') - - try: - output = subprocess.check_output(['node', '--eval', 'console.log("42")']) - if output.strip() == '42': - - return - except (subprocess.CalledProcessError, OSError): - pass - Die('Node.js not found. Try "apt-get install nodejs"or follow the install instructions at https://github.com/ampproject/amphtml/blob/master/validator/README.md#installation') + try: + output = subprocess.check_output(['node', '--eval', 'console.log("42")']) + if output.strip() == '42': + return + except (subprocess.CalledProcessError, OSError): + pass + Die('Node.js not found. Try "apt-get install nodejs" or follow the install instructions at https://github.com/ampproject/amphtml/blob/master/validator/README.md#installation') def CheckPrereqs(): From 85e6f789068bf5bf17a664f1e56606da60ea13d1 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 10:45:50 +0000 Subject: [PATCH 3/9] Resolving merge --- validator/build.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/validator/build.py b/validator/build.py index 54270ad3d295..dd869f58d203 100755 --- a/validator/build.py +++ b/validator/build.py @@ -180,6 +180,7 @@ def GenValidatorProtoascii(out_dir): logging.info('... done') + def GenValidatorProtoGeneratedJs(out_dir): """Calls validator_gen_js to generate validator-proto-generated.js. @@ -203,8 +204,8 @@ def GenValidatorProtoGeneratedJs(out_dir): validator_gen_js.GenerateValidatorGeneratedJs( specfile=None, validator_pb2=validator_pb2, - generate_proto_only=True, - generate_spec_only=False, + generate_proto_only=False, + generate_spec_only=True, text_format=text_format, html_format=None, descriptor=descriptor, @@ -239,8 +240,8 @@ def GenValidatorProtoGeneratedJs(out_dir): validator_gen_js.GenerateValidatorGeneratedJs( specfile=None, validator_pb2=validator_pb2, - generate_proto_only=False, - generate_spec_only=True, + generate_proto_only=True, + generate_spec_only=False, text_format=text_format, html_format=None, descriptor=descriptor, @@ -654,8 +655,6 @@ def RunTests(update_tests, out_dir): - - def Main(parsed_args): """The main method, which executes all build steps and runs the tests.""" logging.basicConfig( From 68ca2296762887a54d9c294e00fcd95aeeafb003 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 10:49:07 +0000 Subject: [PATCH 4/9] Resolving merge --- validator/build.py | 70 +++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/validator/build.py b/validator/build.py index dd869f58d203..21f970fe708e 100755 --- a/validator/build.py +++ b/validator/build.py @@ -180,7 +180,6 @@ def GenValidatorProtoascii(out_dir): logging.info('... done') - def GenValidatorProtoGeneratedJs(out_dir): """Calls validator_gen_js to generate validator-proto-generated.js. @@ -217,8 +216,8 @@ def GenValidatorProtoGeneratedJs(out_dir): logging.info('... done') -def GenValidatorProtoGeneratedJs(out_dir): - """Calls validator_gen_js to generate validator-proto-generated.js. +def GenValidatorGeneratedJs(out_dir): + """Calls validator_gen_js to generate validator-generated.js. Args: out_dir: directory name of the output directory. Must not have slashes, @@ -238,24 +237,23 @@ def GenValidatorProtoGeneratedJs(out_dir): # pylint: enable=g-import-not-at-top out = [] validator_gen_js.GenerateValidatorGeneratedJs( - specfile=None, + specfile='%s/validator.protoascii' % out_dir, validator_pb2=validator_pb2, - generate_proto_only=True, - generate_spec_only=False, + generate_proto_only=False, + generate_spec_only=True, text_format=text_format, html_format=None, descriptor=descriptor, out=out) out.append('') - f = open('%s/validator-proto-generated.js' % out_dir, 'w') + f = open('%s/validator-generated.js' % out_dir, 'w') f.write('\n'.join(out)) f.close() logging.info('... done') -# Similar to GenValidatorGeneratedJS(), except calls out for PHP generation -def GenValidatorGeneratedPHP(out_dir): - """Calls validator_gen to generate validator-generated.php. +def GenValidatorProtoGeneratedJs(out_dir): + """Calls validator_gen_js to generate validator-proto-generated.js. Args: out_dir: directory name of the output directory. Must not have slashes, @@ -267,24 +265,32 @@ def GenValidatorGeneratedPHP(out_dir): # These imports happen late, within this method because they don't necessarily # exist when the module starts running, and the ones that probably do # are checked by CheckPrereqs. + # pylint: disable=g-import-not-at-top from google.protobuf import text_format from google.protobuf import descriptor from dist import validator_pb2 - import validator_gen_php + import validator_gen_js + # pylint: enable=g-import-not-at-top out = [] - validator_gen_php.GenerateValidatorGeneratedPHP(specfile='%s/validator.protoascii' % out_dir, - validator_pb2=validator_pb2, - text_format=text_format, - descriptor=descriptor, - out=out) + validator_gen_js.GenerateValidatorGeneratedJs( + specfile=None, + validator_pb2=validator_pb2, + generate_proto_only=True, + generate_spec_only=False, + text_format=text_format, + html_format=None, + descriptor=descriptor, + out=out) out.append('') - f = open('%s/validator-generated.php' % out_dir, 'w') + f = open('%s/validator-proto-generated.js' % out_dir, 'w') f.write('\n'.join(out)) f.close() logging.info('... done') -def GenValidatorGeneratedJs(out_dir): - """Calls validator_gen_js to generate validator-generated.js. + +# Similar to GenValidatorGeneratedJS(), except calls out for PHP generation +def GenValidatorGeneratedPHP(out_dir): + """Calls validator_gen to generate validator-generated.php. Args: out_dir: directory name of the output directory. Must not have slashes, @@ -296,28 +302,23 @@ def GenValidatorGeneratedJs(out_dir): # These imports happen late, within this method because they don't necessarily # exist when the module starts running, and the ones that probably do # are checked by CheckPrereqs. - # pylint: disable=g-import-not-at-top from google.protobuf import text_format from google.protobuf import descriptor from dist import validator_pb2 - import validator_gen_js - # pylint: enable=g-import-not-at-top + import validator_gen_php out = [] - validator_gen_js.GenerateValidatorGeneratedJs( - specfile='%s/validator.protoascii' % out_dir, - validator_pb2=validator_pb2, - generate_proto_only=False, - generate_spec_only=True, - text_format=text_format, - html_format=None, - descriptor=descriptor, - out=out) + validator_gen_php.GenerateValidatorGeneratedPHP(specfile='%s/validator.protoascii' % out_dir, + validator_pb2=validator_pb2, + text_format=text_format, + descriptor=descriptor, + out=out) out.append('') - f = open('%s/validator-generated.js' % out_dir, 'w') + f = open('%s/validator-generated.php' % out_dir, 'w') f.write('\n'.join(out)) f.close() logging.info('... done') + def CompileWithClosure(js_files, definitions, entry_points, output_file): """Compiles the arguments with the Closure compiler for transpilation to ES5. @@ -654,7 +655,6 @@ def RunTests(update_tests, out_dir): logging.info('... success') - def Main(parsed_args): """The main method, which executes all build steps and runs the tests.""" logging.basicConfig( @@ -666,10 +666,10 @@ def Main(parsed_args): SetupOutDir(out_dir='dist') GenValidatorProtoascii(out_dir='dist') GenValidatorPb2Py(out_dir='dist') - GenValidatorGeneratedPHP(out_dir='dist') - GenValidatorGeneratedMd(out_dir='dist') GenValidatorProtoGeneratedJs(out_dir='dist') GenValidatorGeneratedJs(out_dir='dist') + GenValidatorGeneratedPHP(out_dir='dist') + GenValidatorGeneratedMd(out_dir='dist') CompileValidatorMinified(out_dir='dist') RunSmokeTest(out_dir='dist') RunIndexTest() From 003b793a07f90010650017d7d194bc0374bdcf75 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 10:53:04 +0000 Subject: [PATCH 5/9] Adding files --- validator/validator_gen_md.py | 218 ++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 validator/validator_gen_md.py diff --git a/validator/validator_gen_md.py b/validator/validator_gen_md.py new file mode 100644 index 000000000000..ecac7eabbc9f --- /dev/null +++ b/validator/validator_gen_md.py @@ -0,0 +1,218 @@ +# +# Copyright 2015 The AMP HTML Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the license. +# +"""Generates validator-generated.md""" + +import os + + +def GenerateValidatorGeneratedMd(specfile, validator_pb2, text_format, out): + """Main method for the markdown generator. + + This method reads the specfile and emits Markdown to sys.stdout. + + Args: + specfile: Path to validator.protoascii. + validator_pb2: The proto2 Python module generated from validator.proto. + text_format: The text_format module from the protobuf package, e.g. + google.protobuf.text_format. + out: a list of lines to output (without the newline characters), to + which this function will append. + """ + out.append('' % + os.path.basename(__file__)) + out.append( + '') + out.append('') + out.append('[Accelerated Mobile Pages Project](https://www.ampproject.org)') + out.append('') + + out.append('# validator.protoascii') + out.append('') + + rules = validator_pb2.ValidatorRules() + text_format.Merge(open(specfile).read(), rules) + if rules.HasField('spec_file_revision'): + out.append('* spec file revision: %d' % rules.spec_file_revision) + if rules.HasField('min_validator_revision_required'): + out.append('* minimum validator revision required: %d' % + rules.min_validator_revision_required) + out.append('') + out.append('Allowed Tags') + out.append('') + out.append('[TOC]') + out.append('') + for (field_desc, field_val) in rules.ListFields(): + if field_desc.name == 'tags': + for tag_spec in field_val: + PrintTagSpec(validator_pb2, tag_spec, out) + + +def GetLayout(validator_pb2, layout_index): + """Helper function that returns the AmpLayout.Layout name for a given index. + + See amp.validator.AmpLayout.Layout in validator.proto for details. + + Args: + validator_pb2: The proto2 Python module generated from validator.proto. + layout_index: integer representing a particular AmpLayout.Layout + Returns: + A string which represents the name for this supported layout. + """ + amp_layout = validator_pb2.DESCRIPTOR.message_types_by_name['AmpLayout'] + layouts = amp_layout.fields_by_name['supported_layouts'].enum_type.values + return layouts[layout_index].name + + +def PrintAmpLayout(validator_pb2, amp_layout, out): + """Prints a Markdown version of the given proto message (AmpLayout). + + See amp.validator.AmpLayout in validator.proto for details of proto message. + + Args: + validator_pb2: The proto2 Python module generated from validator.proto. + amp_layout: The AmpLayout message. + out: A list of lines to output (without newline characters), to which this + function will append. + """ + for layout in amp_layout.supported_layouts: + out.append('* %s' % UnicodeEscape(GetLayout(validator_pb2, layout))) + if amp_layout.defines_default_width: + out.append('* Defines Default Width') + if amp_layout.defines_default_height: + out.append('* Defines Default Height') + + +def PrintAttrSpec(attr_spec, out): + """Prints a Markdown version of the given proto message (AttrSpec). + + See amp.validator.AttrSpec in validator.proto for details of proto message. + + Args: + attr_spec: The AttrSpec message. + out: A list of lines to output (without newline characters), to which this + function will append. + """ + out.append('* %s' % UnicodeEscape(attr_spec.name)) + if attr_spec.alternative_names: + out.append(' * Alternative Names: %s' % + RepeatedFieldToString(attr_spec.alternative_names)) + if attr_spec.mandatory: + out.append(' * Mandatory') + if attr_spec.mandatory_oneof: + out.append(' * Mandatory One of: %s' % attr_spec.mandatory_oneof) + if attr_spec.value: + out.append(' * Required Value: %s' % attr_spec.value) + + +def PrintTagSpec(validator_pb2, tag_spec, out): + """Prints a Markdown version of the given proto message (TagSpec). + + See amp.validator.TagSpec in validator.proto for details of proto message. + + Args: + validator_pb2: The proto2 Python module generated from validator.proto. + tag_spec: The TagSpec message. + out: A list of lines to output (without newline characters), to which this + function will append. + """ + header = '## %s' % UnicodeEscape(tag_spec.tag_name) + if tag_spec.spec_name and (tag_spec.tag_name != tag_spec.spec_name): + header += ': %s' % UnicodeEscape(tag_spec.spec_name) + if tag_spec.deprecation: + header += ' (DEPRECATED)' + header += '{#%s_%s}' % (UnicodeEscape(tag_spec.tag_name).replace(' ', '_'), + UnicodeEscape(tag_spec.spec_name).replace(' ', '_')) + out.append('') + out.append(header) + out.append('') + if tag_spec.deprecation: + out.append('This is deprecated.') + out.append('Please see [%s](%s)' % + (UnicodeEscape(tag_spec.deprecation), tag_spec.deprecation_url)) + out.append('') + if tag_spec.mandatory: + out.append('* Mandatory') + if tag_spec.mandatory_alternatives: + out.append('* Mandatory Alternatives: %s' % + UnicodeEscape(tag_spec.mandatory_alternatives)) + if tag_spec.unique: + out.append('* Unique') + if tag_spec.mandatory_parent: + out.append('* Mandatory Parent: %s' % + UnicodeEscape(tag_spec.mandatory_parent)) + if tag_spec.mandatory_ancestor: + out.append('* Mandatory Ancestor: %s' % + UnicodeEscape(tag_spec.mandatory_ancestor)) + if tag_spec.mandatory_ancestor_suggested_alternative: + out.append('* Mandatory Ancestor Alternative: %s' % + UnicodeEscape(tag_spec.mandatory_ancestor_suggested_alternative)) + if tag_spec.disallowed_ancestor: + out.append('* Disallowed Ancestor: %s' % + RepeatedFieldToString(tag_spec.disallowed_ancestor)) + if tag_spec.also_requires_tag: + out.append('* Also Requires: %s' % + RepeatedFieldToString(tag_spec.also_requires_tag)) + if tag_spec.attrs: + out.append('') + out.append('Allowed Attributes:') + out.append('') + for attr_spec in tag_spec.attrs: + PrintAttrSpec(attr_spec, out) + if (tag_spec.amp_layout.supported_layouts or + tag_spec.amp_layout.defines_default_width or + tag_spec.amp_layout.defines_default_height): + out.append('') + out.append('Allowed Layouts:') + out.append('') + PrintAmpLayout(validator_pb2, tag_spec.amp_layout, out) + if tag_spec.spec_url: + out.append('') + out.append('[Spec](%s)' % tag_spec.spec_url) + out.append('') + + +def RepeatedFieldToString(field): + """Helper function which converts a list into an escaped string. + + Args: + field: A list of strings. + Returns: + A string, segmented by commas. + """ + return ', '.join([UnicodeEscape(s) for s in field]) + + +def UnderscoreToTitleCase(under_score): + """Helper function which converts under_score names to TitleCase. + + Args: + under_score: A name, segmented by under_scores. + Returns: + A name, segmented as TitleCase. + """ + segments = under_score.split('_') + return ' '.join([s.title() for s in segments]) + + +def UnicodeEscape(string): + """Helper function which escapes unicode characters. + + Args: + string: A string which may contain unicode characters. + Returns: + An escaped string. + """ + return ('' + string).encode('unicode-escape') From 28de79532f68125dfcd457b0e50322a41cb7784a Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 11:17:11 +0000 Subject: [PATCH 6/9] Resolving merge --- validator/build.py | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/validator/build.py b/validator/build.py index 21f970fe708e..35057f34a97e 100755 --- a/validator/build.py +++ b/validator/build.py @@ -252,42 +252,6 @@ def GenValidatorGeneratedJs(out_dir): logging.info('... done') -def GenValidatorProtoGeneratedJs(out_dir): - """Calls validator_gen_js to generate validator-proto-generated.js. - - Args: - out_dir: directory name of the output directory. Must not have slashes, - dots, etc. - """ - logging.info('entering ...') - assert re.match(r'^[a-zA-Z_\-0-9]+$', out_dir), 'bad out_dir: %s' % out_dir - - # These imports happen late, within this method because they don't necessarily - # exist when the module starts running, and the ones that probably do - # are checked by CheckPrereqs. - # pylint: disable=g-import-not-at-top - from google.protobuf import text_format - from google.protobuf import descriptor - from dist import validator_pb2 - import validator_gen_js - # pylint: enable=g-import-not-at-top - out = [] - validator_gen_js.GenerateValidatorGeneratedJs( - specfile=None, - validator_pb2=validator_pb2, - generate_proto_only=True, - generate_spec_only=False, - text_format=text_format, - html_format=None, - descriptor=descriptor, - out=out) - out.append('') - f = open('%s/validator-proto-generated.js' % out_dir, 'w') - f.write('\n'.join(out)) - f.close() - logging.info('... done') - - # Similar to GenValidatorGeneratedJS(), except calls out for PHP generation def GenValidatorGeneratedPHP(out_dir): """Calls validator_gen to generate validator-generated.php. @@ -680,9 +644,10 @@ def Main(parsed_args): CompileAmp4AdsParseCssTestMinified(out_dir='dist') CompileKeyframesParseCssTestMinified(out_dir='dist') CompileParseSrcsetTestMinified(out_dir='dist') + CompileValidatorLightTestMinified(out_dir='dist') GenerateTestRunner(out_dir='dist') RunTests(update_tests=parsed_args.update_tests, out_dir='dist') - CreateWebuiAppengineDist(out_dir='dist') + #CreateWebuiAppengineDist(out_dir='dist') if __name__ == '__main__': parser = argparse.ArgumentParser( From 69a6390d86a06645f4c4a6c49e7fa688613c9dd1 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 11:19:29 +0000 Subject: [PATCH 7/9] Resolving merge --- validator/build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/build.py b/validator/build.py index 35057f34a97e..bccefa36130f 100755 --- a/validator/build.py +++ b/validator/build.py @@ -203,8 +203,8 @@ def GenValidatorProtoGeneratedJs(out_dir): validator_gen_js.GenerateValidatorGeneratedJs( specfile=None, validator_pb2=validator_pb2, - generate_proto_only=False, - generate_spec_only=True, + generate_proto_only=True, + generate_spec_only=False, text_format=text_format, html_format=None, descriptor=descriptor, From 90544523cd4e9d8ede6a781920e19c59d9c55b59 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 11:36:26 +0000 Subject: [PATCH 8/9] Resolving merge --- validator/build.py | 1 - validator/validator_gen_md.py | 218 ---------------------------------- 2 files changed, 219 deletions(-) delete mode 100644 validator/validator_gen_md.py diff --git a/validator/build.py b/validator/build.py index bccefa36130f..de768e20cc1b 100755 --- a/validator/build.py +++ b/validator/build.py @@ -633,7 +633,6 @@ def Main(parsed_args): GenValidatorProtoGeneratedJs(out_dir='dist') GenValidatorGeneratedJs(out_dir='dist') GenValidatorGeneratedPHP(out_dir='dist') - GenValidatorGeneratedMd(out_dir='dist') CompileValidatorMinified(out_dir='dist') RunSmokeTest(out_dir='dist') RunIndexTest() diff --git a/validator/validator_gen_md.py b/validator/validator_gen_md.py deleted file mode 100644 index ecac7eabbc9f..000000000000 --- a/validator/validator_gen_md.py +++ /dev/null @@ -1,218 +0,0 @@ -# -# Copyright 2015 The AMP HTML Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the license. -# -"""Generates validator-generated.md""" - -import os - - -def GenerateValidatorGeneratedMd(specfile, validator_pb2, text_format, out): - """Main method for the markdown generator. - - This method reads the specfile and emits Markdown to sys.stdout. - - Args: - specfile: Path to validator.protoascii. - validator_pb2: The proto2 Python module generated from validator.proto. - text_format: The text_format module from the protobuf package, e.g. - google.protobuf.text_format. - out: a list of lines to output (without the newline characters), to - which this function will append. - """ - out.append('' % - os.path.basename(__file__)) - out.append( - '') - out.append('') - out.append('[Accelerated Mobile Pages Project](https://www.ampproject.org)') - out.append('') - - out.append('# validator.protoascii') - out.append('') - - rules = validator_pb2.ValidatorRules() - text_format.Merge(open(specfile).read(), rules) - if rules.HasField('spec_file_revision'): - out.append('* spec file revision: %d' % rules.spec_file_revision) - if rules.HasField('min_validator_revision_required'): - out.append('* minimum validator revision required: %d' % - rules.min_validator_revision_required) - out.append('') - out.append('Allowed Tags') - out.append('') - out.append('[TOC]') - out.append('') - for (field_desc, field_val) in rules.ListFields(): - if field_desc.name == 'tags': - for tag_spec in field_val: - PrintTagSpec(validator_pb2, tag_spec, out) - - -def GetLayout(validator_pb2, layout_index): - """Helper function that returns the AmpLayout.Layout name for a given index. - - See amp.validator.AmpLayout.Layout in validator.proto for details. - - Args: - validator_pb2: The proto2 Python module generated from validator.proto. - layout_index: integer representing a particular AmpLayout.Layout - Returns: - A string which represents the name for this supported layout. - """ - amp_layout = validator_pb2.DESCRIPTOR.message_types_by_name['AmpLayout'] - layouts = amp_layout.fields_by_name['supported_layouts'].enum_type.values - return layouts[layout_index].name - - -def PrintAmpLayout(validator_pb2, amp_layout, out): - """Prints a Markdown version of the given proto message (AmpLayout). - - See amp.validator.AmpLayout in validator.proto for details of proto message. - - Args: - validator_pb2: The proto2 Python module generated from validator.proto. - amp_layout: The AmpLayout message. - out: A list of lines to output (without newline characters), to which this - function will append. - """ - for layout in amp_layout.supported_layouts: - out.append('* %s' % UnicodeEscape(GetLayout(validator_pb2, layout))) - if amp_layout.defines_default_width: - out.append('* Defines Default Width') - if amp_layout.defines_default_height: - out.append('* Defines Default Height') - - -def PrintAttrSpec(attr_spec, out): - """Prints a Markdown version of the given proto message (AttrSpec). - - See amp.validator.AttrSpec in validator.proto for details of proto message. - - Args: - attr_spec: The AttrSpec message. - out: A list of lines to output (without newline characters), to which this - function will append. - """ - out.append('* %s' % UnicodeEscape(attr_spec.name)) - if attr_spec.alternative_names: - out.append(' * Alternative Names: %s' % - RepeatedFieldToString(attr_spec.alternative_names)) - if attr_spec.mandatory: - out.append(' * Mandatory') - if attr_spec.mandatory_oneof: - out.append(' * Mandatory One of: %s' % attr_spec.mandatory_oneof) - if attr_spec.value: - out.append(' * Required Value: %s' % attr_spec.value) - - -def PrintTagSpec(validator_pb2, tag_spec, out): - """Prints a Markdown version of the given proto message (TagSpec). - - See amp.validator.TagSpec in validator.proto for details of proto message. - - Args: - validator_pb2: The proto2 Python module generated from validator.proto. - tag_spec: The TagSpec message. - out: A list of lines to output (without newline characters), to which this - function will append. - """ - header = '## %s' % UnicodeEscape(tag_spec.tag_name) - if tag_spec.spec_name and (tag_spec.tag_name != tag_spec.spec_name): - header += ': %s' % UnicodeEscape(tag_spec.spec_name) - if tag_spec.deprecation: - header += ' (DEPRECATED)' - header += '{#%s_%s}' % (UnicodeEscape(tag_spec.tag_name).replace(' ', '_'), - UnicodeEscape(tag_spec.spec_name).replace(' ', '_')) - out.append('') - out.append(header) - out.append('') - if tag_spec.deprecation: - out.append('This is deprecated.') - out.append('Please see [%s](%s)' % - (UnicodeEscape(tag_spec.deprecation), tag_spec.deprecation_url)) - out.append('') - if tag_spec.mandatory: - out.append('* Mandatory') - if tag_spec.mandatory_alternatives: - out.append('* Mandatory Alternatives: %s' % - UnicodeEscape(tag_spec.mandatory_alternatives)) - if tag_spec.unique: - out.append('* Unique') - if tag_spec.mandatory_parent: - out.append('* Mandatory Parent: %s' % - UnicodeEscape(tag_spec.mandatory_parent)) - if tag_spec.mandatory_ancestor: - out.append('* Mandatory Ancestor: %s' % - UnicodeEscape(tag_spec.mandatory_ancestor)) - if tag_spec.mandatory_ancestor_suggested_alternative: - out.append('* Mandatory Ancestor Alternative: %s' % - UnicodeEscape(tag_spec.mandatory_ancestor_suggested_alternative)) - if tag_spec.disallowed_ancestor: - out.append('* Disallowed Ancestor: %s' % - RepeatedFieldToString(tag_spec.disallowed_ancestor)) - if tag_spec.also_requires_tag: - out.append('* Also Requires: %s' % - RepeatedFieldToString(tag_spec.also_requires_tag)) - if tag_spec.attrs: - out.append('') - out.append('Allowed Attributes:') - out.append('') - for attr_spec in tag_spec.attrs: - PrintAttrSpec(attr_spec, out) - if (tag_spec.amp_layout.supported_layouts or - tag_spec.amp_layout.defines_default_width or - tag_spec.amp_layout.defines_default_height): - out.append('') - out.append('Allowed Layouts:') - out.append('') - PrintAmpLayout(validator_pb2, tag_spec.amp_layout, out) - if tag_spec.spec_url: - out.append('') - out.append('[Spec](%s)' % tag_spec.spec_url) - out.append('') - - -def RepeatedFieldToString(field): - """Helper function which converts a list into an escaped string. - - Args: - field: A list of strings. - Returns: - A string, segmented by commas. - """ - return ', '.join([UnicodeEscape(s) for s in field]) - - -def UnderscoreToTitleCase(under_score): - """Helper function which converts under_score names to TitleCase. - - Args: - under_score: A name, segmented by under_scores. - Returns: - A name, segmented as TitleCase. - """ - segments = under_score.split('_') - return ' '.join([s.title() for s in segments]) - - -def UnicodeEscape(string): - """Helper function which escapes unicode characters. - - Args: - string: A string which may contain unicode characters. - Returns: - An escaped string. - """ - return ('' + string).encode('unicode-escape') From d11558ef72a61506b0238ba7addd4ba7131cabb4 Mon Sep 17 00:00:00 2001 From: Marcelo Vani Date: Tue, 20 Nov 2018 11:54:07 +0000 Subject: [PATCH 9/9] Resolving merge --- validator/build.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/validator/build.py b/validator/build.py index de768e20cc1b..34d25a1c00b8 100755 --- a/validator/build.py +++ b/validator/build.py @@ -325,7 +325,7 @@ def CompileValidatorMinified(out_dir): 'engine/parse-url.js', 'engine/tokenize-css.js', '%s/validator-generated.js' % out_dir, '%s/validator-proto-generated.js' % out_dir, - 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/validator-full.js', + 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/amp4ads-parse-css.js', 'engine/keyframes-parse-css.js', 'engine/htmlparser-interface.js' ], @@ -410,7 +410,7 @@ def CompileValidatorTestMinified(out_dir): 'engine/parse-url.js', 'engine/tokenize-css.js', '%s/validator-generated.js' % out_dir, '%s/validator-proto-generated.js' % out_dir, - 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/validator-full.js', + 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/amp4ads-parse-css.js', 'engine/keyframes-parse-css.js', 'engine/htmlparser-interface.js', 'engine/validator_test.js' ], @@ -420,21 +420,6 @@ def CompileValidatorTestMinified(out_dir): logging.info('... success') -def CompileValidatorLightTestMinified(out_dir): - logging.info('entering ...') - CompileWithClosure( - js_files=['engine/htmlparser.js', 'engine/parse-css.js', 'engine/parse-srcset.js', - 'engine/tokenize-css.js', '%s/validator-generated.js' % out_dir, - '%s/validator-proto-generated.js' % out_dir, - 'engine/validator-in-browser.js', 'engine/validator.js', 'engine/validator-light.js', - 'engine/htmlparser-interface.js', 'engine/dom-walker.js', - 'engine/validator-light_test.js'], - definitions=[], - entry_points=['amp.validator.ValidatorTest'], - output_file='%s/validator-light_test_minified.js' % out_dir) - logging.info('... success') - - def CompileHtmlparserTestMinified(out_dir): """Runs closure compiler for htmlparser_test.js. @@ -643,10 +628,8 @@ def Main(parsed_args): CompileAmp4AdsParseCssTestMinified(out_dir='dist') CompileKeyframesParseCssTestMinified(out_dir='dist') CompileParseSrcsetTestMinified(out_dir='dist') - CompileValidatorLightTestMinified(out_dir='dist') GenerateTestRunner(out_dir='dist') RunTests(update_tests=parsed_args.update_tests, out_dir='dist') - #CreateWebuiAppengineDist(out_dir='dist') if __name__ == '__main__': parser = argparse.ArgumentParser(