This page exists to communicate the design behind the minimal prototype of OEP-37: Dev Data.

Visualization of whole system

Prototype implementation

You can find implementation details in this expand. PR links also included in expand.

Make command

dev.load_data:
	python load_data.py ${data_spec_top_path}

Python script called by make command

#!/usr/bin/env python3

import yaml
import sys
import subprocess


def main(input_yaml_path):
    with open(input_yaml_path, 'r') as f:
        top_data_spec_yaml = yaml.safe_load(f)
        for data_spec_path in top_data_spec_yaml:
            ida_name = data_spec_path['ida_name']
            ida_data_spec_yaml = data_spec_path['data_spec_path']
            print(f"Creating test data in {ida_name} based on {ida_data_spec_yaml}")
            if ida_name == "lms" or ida_name == "cms":
                subprocess.run(f"docker-compose exec -T {ida_name} bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && python manage.py {ida_name} load_data --data-file-path {ida_data_spec_yaml}'", shell=True)
            else:
                subprocess.call(f"docker-compose exec -T {ida_name} bash -c 'source /edx/app/{ida_name}/{ida_name}_env && cd /edx/app/{ida_name}/{ida_name}/ && python manage.py load_data --data-file-path {ida_data_spec_yaml}'", shell=True)


if __name__ == "__main__":
    if len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        print("Path to data spec yaml not specified")

Yaml files

Top level Yaml file

- ida_name: lms
  data_spec_path: openedx/core/djangoapps/util/management/commands/test_command.yaml
- ida_name: ecommerce
  data_spec_path: ecommerce/core/management/commands/test_data.yaml
- ida_name: lms
  data_spec_path: openedx/core/djangoapps/util/management/commands/test_command.yaml

LMS Yaml file

users:
  - username: verified
    email: verified@example.com
  - username: robot1
    email: robot1@example.com
enrollments:
  - username: verified
    course_id: 'course-v1:edX+DemoX+Demo_Course'
    mode: verified
  - username: robot1
    course_id: 'course-v1:edX+DemoX+Demo_Course'
    mode: verified

LMS Management Command

#!/usr/bin/env python3
from django.core.management.base import BaseCommand
from django.db.utils import IntegrityError
from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory
from common.djangoapps.course_modes.models import CourseMode
from django.contrib.auth import get_user_model
import yaml
import codecs

from lms.djangoapps.verify_student.tests.factories import SoftwareSecurePhotoVerificationFactory

class Command(BaseCommand):
    """
    Use to populate your database with some essential basic data
    """

    def add_arguments(self, parser):
        parser.add_argument('--data-file-path', type=str, required=True, help="Path to file where your data is specified.")


    def handle(self, *args, **options):
        with open(options["data_file_path"], 'r') as f:
            data_spec = yaml.safe_load(f)
        for user_spec in data_spec.get('users', []):
            self.create_user(user_spec)
        for enrollment_spec in data_spec.get('enrollments', []):
            self.create_enrollment(enrollment_spec)

    def create_user(self, user):
        """
        Use to create users in your database.
        """
        return UserFactory.create(**user)

    def create_enrollment(self, enrollment_spec):
        """
        Use to create enrollments in your database.
        """
        User = get_user_model()
        try:
            user = User.objects.get(username=enrollment_spec['username'])
        except User.DoesNotExist:
            raise exception(f"User:{enrollment_spec['username']} not created before trying to create enrollment")
        if enrollment_spec['mode'] in CourseMode.VERIFIED_MODES:
            verfication = SoftwareSecurePhotoVerificationFactory(user=user)
        enrollment = CourseEnrollmentFactory(user=user, course_id=enrollment_spec['course_id'])
        if enrollment.mode != enrollment_spec['mode']:
            enrollment.mode = enrollment_spec['mode']
            enrollment.save()
        return enrollment

Sample Factory

class UserFactory(DjangoModelFactory):  # lint-amnesty, pylint: disable=missing-class-docstring
    class Meta:
        model = User
        django_get_or_create = ('email', 'username')

    _DEFAULT_PASSWORD = 'test'

    username = factory.Sequence('robot{}'.format)
    email = factory.Sequence('robot+test+{}@edx.org'.format)

#.........
#stuff skipped
#.........

    @factory.post_generation
    def profile(obj, create, extracted, **kwargs):  # pylint: disable=unused-argument, missing-function-docstring
        if create:
            obj.save()
            return UserProfileFactory.create(user=obj, **kwargs)
        elif kwargs:
            raise Exception("Cannot build a user profile without saving the user")
        else:
            return None
#.........
#stuff skipped
#.........

Ecommerce PR

https://github.com/edx/ecommerce/pull/3360

edx-platform PR

https://github.com/edx/edx-platform/pull/27043

Devstack PR

https://github.com/edx/devstack/pull/700

How to use implementation:

Design Decisions

Open Design Decisions

Possible Future Roadmap

Different stages