Skip to content

Add data files to the wheel package#1727

Merged
masci merged 4 commits into
masterfrom
ofek/fix-packaging
Jun 25, 2018
Merged

Add data files to the wheel package#1727
masci merged 4 commits into
masterfrom
ofek/fix-packaging

Conversation

@ofek

@ofek ofek commented Jun 12, 2018

Copy link
Copy Markdown
Contributor

What does this PR do?

This moves the yaml configs to a location that can be picked up by pip/wheel.

Motivation

They aren't being included currently.

Additional Notes

Script (Python 3 only)

import os
import re
from glob import iglob
from itertools import chain
from shutil import move

CORE_DIR = r'C:\Users\Ofek\Desktop\code\integrations-core'
BASE_CHECKS = {'datadog_checks_base', 'datadog_checks_tests_helper'}
URL_PATTERN = 'https?://github.com/DataDog/integrations-core/blob/master{}'
URL_ROOT = URL_PATTERN.replace('?', '', 1)
MANIFEST_JMX = """\
include README.md
include manifest.json
graft datadog_checks
"""
MANIFEST_TEMPLATE = """\
graft datadog_checks
graft {test_dir}

include MANIFEST.in
include README.md
include requirements.in
include requirements.txt
include requirements-dev.txt
include manifest.json

global-exclude *.py[cod] __pycache__
"""


def get_setup_file(check_name):
    f = os.path.join(CORE_DIR, check_name, 'setup.py')
    if os.path.isfile(f):
        return f


def get_data_dir(check_name):
    return os.path.join(CORE_DIR, check_name, 'datadog_checks', check_name, 'data')


def get_relative_url_path(path):
    return path.replace(CORE_DIR, '').replace(os.path.sep, '/')


def main():
    for check_name in os.listdir(CORE_DIR):
        setup_file = get_setup_file(check_name)
        if setup_file and check_name not in BASE_CHECKS:
            check_dir = os.path.join(CORE_DIR, check_name)

            # Create new data directory
            data_dir = get_data_dir(check_name)
            if not os.path.isdir(data_dir):
                os.makedirs(data_dir)

            # Move package data to new location
            data_files = [
                (old_data_file, move(old_data_file, data_dir))
                for old_data_file in chain(
                    iglob(os.path.join(check_dir, 'auto_conf.yaml')),
                    iglob(os.path.join(check_dir, 'conf.yaml.*')),
                    iglob(os.path.join(check_dir, 'metrics.yaml')),
                )
            ]

            # Ensure we specify package data
            if os.path.isdir(os.path.join(check_dir, 'test')):
                with open(os.path.join(check_dir, 'MANIFEST.in'), 'w') as f:
                    f.write(MANIFEST_TEMPLATE.format(test_dir='test'))
            elif os.path.isdir(os.path.join(check_dir, 'tests')):
                with open(os.path.join(check_dir, 'MANIFEST.in'), 'w') as f:
                    f.write(MANIFEST_TEMPLATE.format(test_dir='tests'))
            else:
                with open(os.path.join(check_dir, 'MANIFEST.in'), 'w') as f:
                    f.write(MANIFEST_JMX)

            # Update URLs in README.md
            readme_file = os.path.join(check_dir, 'README.md')
            if os.path.isfile(readme_file):
                with open(readme_file, 'r', encoding='utf-8') as f:
                    readme_contents = f.read()

                for old_data_file, new_data_file in data_files:
                    readme_contents = re.sub(
                        URL_PATTERN.format(get_relative_url_path(old_data_file)),
                        URL_ROOT.format(get_relative_url_path(new_data_file)),
                        readme_contents
                    )

                with open(readme_file, 'w', encoding='utf-8') as f:
                    f.write(readme_contents)

            # Ensure we include package data in build
            with open(setup_file, 'r', encoding='utf-8') as f:
                setup_lines = f.readlines()

            for i, line in enumerate(reversed(setup_lines), 1):
                if line.strip().startswith('package_data='):
                    del setup_lines[len(setup_lines) - i]
                    break

            with open(setup_file, 'w', encoding='utf-8') as f:
                f.writelines(setup_lines)

if __name__ == '__main__':
    main()

@ofek ofek added this to the 6.4 milestone Jun 12, 2018
@ofek
ofek requested a review from a team as a code owner June 12, 2018 15:31
@ofek
ofek force-pushed the ofek/fix-packaging branch 4 times, most recently from 45f532c to d8abd17 Compare June 14, 2018 17:58
@ofek
ofek requested a review from a team as a code owner June 14, 2018 17:58
@ofek
ofek force-pushed the ofek/fix-packaging branch from d8abd17 to 755fe01 Compare June 24, 2018 23:07
masci
masci previously approved these changes Jun 25, 2018

@masci masci left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤞

@masci
masci merged commit d46740c into master Jun 25, 2018
@masci
masci deleted the ofek/fix-packaging branch June 25, 2018 14:05
@masci masci changed the title properly package data files for wheels Add data files to the wheel package Jun 26, 2018
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants