Page Comparison

import os
import yaml
from stevedore import named

test_config = """
annotations:
    pii:
        - ".. pii::"
        - ".. pii_types::":
            - id
            - name
            - other
        - ".. pii_retirement::":
            - retained
            - local_api
            - consumer_api
            - third_party
    nopii: ".. no_pii::"

extensions:
    python:
        - py
    javascript:
        - js
        - jsx
"""


def load_failed_handler(*args, **kwargs):
    """
    Callback for when we fail to load an extension, otherwise it fails silently
    """
    print(args)
    print(kwargs)


def search(ext, file_handle, file_extensions_map, filename_extension):
    """
    Executes a search on the given file, only if it is configured for this
    extension
    """
    if filename_extension not in file_extensions_map[ext.name]:
        print('{} does not support {}. Skipping.'.format(ext.name, filename_extension))
        return (ext.name, [])

    return ext.name, ext.obj.search(file_handle)


if __name__ == '__main__':
    config = yaml.load(test_config)

    print(config)

    # These are the names of all of our configured extensions
    configured_extension_names = config['extensions'].keys()

    print(configured_extension_names)

    # Load Stevedore extensions that we are configured for (and only those)
    mgr = named.NamedExtensionManager(
        names=configured_extension_names,
        namespace='annotation_finder.searchers',
        invoke_on_load=True,
        on_load_failure_callback=load_failed_handler,
        invoke_args=(config['annotations'],),  # This is temporary
    )

    # Output all found extension entry points (whether or not they were loaded)
    print(mgr.list_entry_points())

    # Output all extensions that were actually able to load
    for extension in mgr.extensions:
        print(extension)

    # Index the results by extension name
    file_extensions_map = {}
    known_extensions = set()
    for extension_name in config['extensions']:
        file_extensions_map[extension_name] = config['extensions'][extension_name]
        known_extensions.update(config['extensions'][extension_name])

    source_path = '/foo/bar/'

    # From here we could begin the actual file searching and reporting...
    # This is not optimized, but without the prints or doing any actual searching 
    # runs all of edx-platform in 1.18 second.
    for root, dirs, files in os.walk(source_path):
        for filename in files:
            filename_extension = os.path.splitext(filename)[1][1:]

            if filename_extension not in known_extensions:
                print("{} is not a known extension, skipping.".format(filename_extension))
                continue

            full_name = os.path.join(root, filename)
            print(full_name)

            with open(full_name, 'r') as file_handle:
                try:
                    # Call get_supported_extensions on all loaded extensions
                    results = mgr.map(search, file_handle, file_extensions_map, filename_extension)
                    print(results)
                except IndexError:
                    # Should we define a catchall in config?
                    print("No file extension in {}, skipping.".format(full_name))