__FILENAME__ = context_processors

import ConfigParser
from django.conf import settings
import logging


log = logging.getLogger(__name__)


# Open and parse the configuration file when the module is initialized
config_file = open(settings.REPO_ROOT / "docs" / "config.ini")
config = ConfigParser.ConfigParser()
config.readfp(config_file)


def doc_url(request=None):  # pylint: disable=unused-argument
    """
    This function is added in the list of TEMPLATE_CONTEXT_PROCESSORS, which is a django setting for
    a tuple of callables that take a request object as their argument and return a dictionary of items
    to be merged into the RequestContext.

    This function returns a dict with get_online_help_info, making it directly available to all mako templates.

    Args:
        request: Currently not used, but is passed by django to context processors.
            May be used in the future for determining the language of choice.
    """

    def get_online_help_info(page_token=None):
        """
        Args:
            page_token: A string that identifies the page for which the help information is requested.
                It should correspond to an option in the docs/config.ini file.  If it doesn't, the "default"
                option is used instead.

        Returns:
            A dict mapping the following items
                * "doc_url" - a string with the url corresponding to the online help location for the given page_token.
                * "pdf_url" - a string with the url corresponding to the location of the PDF help file.
        """

        def get_config_value_with_default(section_name, option, default_option="default"):
            """
            Args:
                section_name: name of the section in the configuration from which the option should be found
                option: name of the configuration option
                default_option: name of the default configuration option whose value should be returned if the
                    requested option is not found
            """
            try:
                return config.get(section_name, option)
            except (ConfigParser.NoOptionError, AttributeError):
                log.debug("Didn't find a configuration option for '%s' section and '%s' option", section_name, option)
                return config.get(section_name, default_option)

        def get_doc_url():
            """
            Returns:
                The URL for the documentation
            """
            return "{url_base}/{language}/{version}/{page_path}".format(
                url_base=config.get("help_settings", "url_base"),
                language=get_config_value_with_default("locales", settings.LANGUAGE_CODE),
                version=config.get("help_settings", "version"),
                page_path=get_config_value_with_default("pages", page_token),
            )

        def get_pdf_url():
            """
            Returns:
                The URL for the PDF document using the pdf_settings and the help_settings (version) in the configuration
            """
            return "{pdf_base}/{version}/{pdf_file}".format(
                pdf_base=config.get("pdf_settings", "pdf_base"),
                version=config.get("help_settings", "version"),
                pdf_file=config.get("pdf_settings", "pdf_file"),
            )

        return {
            "doc_url": get_doc_url(),
            "pdf_url": get_pdf_url(),
        }

    return {'get_online_help_info': get_online_help_info}

########NEW FILE########
__FILENAME__ = course_info_model
"""
Views for viewing, adding, updating and deleting course updates.

Current db representation:
{
    "_id" : locationjson,
    "definition" : {
        "data" : "<ol>[<li><h2>date</h2>content</li>]</ol>"},
        "items" : [{"id": ID, "date": DATE, "content": CONTENT}]
        "metadata" : ignored
    }
}
"""

import re
import logging

from django.http import HttpResponseBadRequest
import django.utils
from django.utils.translation import ugettext as _
from lxml import html, etree

from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.html_module import CourseInfoModule

# # This should be in a class which inherits from XmlDescriptor
log = logging.getLogger(__name__)


def get_course_updates(location, provided_id):
    """
    Retrieve the relevant course_info updates and unpack into the model which the client expects:
    [{id : index, date : string, content : html string}]
    """
    try:
        course_updates = modulestore('direct').get_item(location)
    except ItemNotFoundError:
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)

    course_update_items = get_course_update_items(course_updates, provided_id)
    return _get_visible_update(course_update_items)


def update_course_updates(location, update, passed_id=None, user=None):
    """
    Either add or update the given course update. It will add it if the passed_id is absent or None. It will update it if
    it has an passed_id which has a valid value. Until updates have distinct values, the passed_id is the location url + an index
    into the html structure.
    """
    try:
        course_updates = modulestore('direct').get_item(location)
    except ItemNotFoundError:
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)

    course_update_items = list(reversed(get_course_update_items(course_updates)))

    if passed_id is not None:
        passed_index = _get_index(passed_id)
        # oldest update at start of list
        if 0 < passed_index <= len(course_update_items):
            course_update_dict = course_update_items[passed_index - 1]
            course_update_dict["date"] = update["date"]
            course_update_dict["content"] = update["content"]
            course_update_items[passed_index - 1] = course_update_dict
        else:
            return HttpResponseBadRequest(_("Invalid course update id."))
    else:
        course_update_dict = {
            "id": len(course_update_items) + 1,
            "date": update["date"],
            "content": update["content"],
            "status": CourseInfoModule.STATUS_VISIBLE
        }
        course_update_items.append(course_update_dict)

    # update db record
    save_course_update_items(location, course_updates, course_update_items, user)
    # remove status key
    if "status" in course_update_dict:
        del course_update_dict["status"]
    return course_update_dict


def _course_info_content(html_parsed):
    """
    Constructs the HTML for the course info update, not including the header.
    """
    if len(html_parsed) == 1:
        # could enforce that update[0].tag == 'h2'
        content = html_parsed[0].tail
    else:
        content = html_parsed[0].tail if html_parsed[0].tail is not None else ""
        content += "\n".join([html.tostring(ele) for ele in html_parsed[1:]])
    return content


def _make_update_dict(update):
    """
    Return course update item as a dictionary with required keys ('id', "date" and "content").
    """
    return {
        "id": update["id"],
        "date": update["date"],
        "content": update["content"],
    }


def _get_visible_update(course_update_items):
    """
    Filter course update items which have status "deleted".
    """
    if isinstance(course_update_items, dict):
        # single course update item
        if course_update_items.get("status") != CourseInfoModule.STATUS_DELETED:
            return _make_update_dict(course_update_items)
        else:
            # requested course update item has been deleted (soft delete)
            return {"error": _("Course update not found."), "status": 404}

    return ([_make_update_dict(update) for update in course_update_items
             if update.get("status") != CourseInfoModule.STATUS_DELETED])


# pylint: disable=unused-argument
def delete_course_update(location, update, passed_id, user):
    """
    Don't delete course update item from db.
    Delete the given course_info update by settings "status" flag to 'deleted'.
    Returns the resulting course_updates.
    """
    if not passed_id:
        return HttpResponseBadRequest()

    try:
        course_updates = modulestore('direct').get_item(location)
    except ItemNotFoundError:
        return HttpResponseBadRequest()

    course_update_items = list(reversed(get_course_update_items(course_updates)))
    passed_index = _get_index(passed_id)

    # delete update item from given index
    if 0 < passed_index <= len(course_update_items):
        course_update_item = course_update_items[passed_index - 1]
        # soft delete course update item
        course_update_item["status"] = CourseInfoModule.STATUS_DELETED
        course_update_items[passed_index - 1] = course_update_item

        # update db record
        save_course_update_items(location, course_updates, course_update_items, user)
        return _get_visible_update(course_update_items)
    else:
        return HttpResponseBadRequest(_("Invalid course update id."))


def _get_index(passed_id=None):
    """
    From the url w/ index appended, get the index.
    """
    if passed_id:
        index_matcher = re.search(r'.*?/?(\d+)$', passed_id)
        if index_matcher:
            return int(index_matcher.group(1))

    # return 0 if no index found
    return 0


def get_course_update_items(course_updates, provided_id=None):
    """
    Returns list of course_updates data dictionaries either from new format if available or
    from old. This function don't modify old data to new data (in db), instead returns data
    in common old dictionary format.
    New Format: {"items" : [{"id": computed_id, "date": date, "content": html-string}],
                 "data": "<ol>[<li><h2>date</h2>content</li>]</ol>"}
    Old Format: {"data": "<ol>[<li><h2>date</h2>content</li>]</ol>"}
    """
    if course_updates and getattr(course_updates, "items", None):
        provided_id = _get_index(provided_id)
        if provided_id and 0 < provided_id <= len(course_updates.items):
            return course_updates.items[provided_id - 1]

        # return list in reversed order (old format: [4,3,2,1]) for compatibility
        return list(reversed(course_updates.items))
    else:
        # old method to get course updates
        # purely to handle free formed updates not done via editor. Actually kills them, but at least doesn't break.
        try:
            course_html_parsed = html.fromstring(course_updates.data)
        except (etree.XMLSyntaxError, etree.ParserError):
            log.error("Cannot parse: " + course_updates.data)
            escaped = django.utils.html.escape(course_updates.data)
            course_html_parsed = html.fromstring("<ol><li>" + escaped + "</li></ol>")

        # confirm that root is <ol>, iterate over <li>, pull out <h2> subs and then rest of val
        course_update_items = []
        provided_id = _get_index(provided_id)
        if course_html_parsed.tag == 'ol':
            # 0 is the newest
            for index, update in enumerate(course_html_parsed):
                if len(update) > 0:
                    content = _course_info_content(update)
                    # make the id on the client be 1..len w/ 1 being the oldest and len being the newest
                    computed_id = len(course_html_parsed) - index
                    payload = {
                        "id": computed_id,
                        "date": update.findtext("h2"),
                        "content": content
                    }
                    if provided_id == 0:
                        course_update_items.append(payload)
                    elif provided_id == computed_id:
                        return payload

        return course_update_items


def _get_html(course_updates_items):
    """
    Method to create course_updates_html from course_updates items
    """
    list_items = []
    for update in reversed(course_updates_items):
        # filter course update items which have status "deleted".
        if update.get("status") != CourseInfoModule.STATUS_DELETED:
            list_items.append(u"<article><h2>{date}</h2>{content}</article>".format(**update))
    return u"<section>{list_items}</section>".format(list_items="".join(list_items))


def save_course_update_items(location, course_updates, course_update_items, user=None):
    """
    Save list of course_updates data dictionaries in new field ("course_updates.items")
    and html related to course update in 'data' ("course_updates.data") field.
    """
    course_updates.items = course_update_items
    course_updates.data = _get_html(course_update_items)

    # update db record
    modulestore('direct').update_item(course_updates, user)

    return course_updates

########NEW FILE########
__FILENAME__ = debug_file_uploader
from django.core.files.uploadhandler import FileUploadHandler
import time


class DebugFileUploader(FileUploadHandler):
    def __init__(self, request=None):
        super(DebugFileUploader, self).__init__(request)
        self.count = 0

    def receive_data_chunk(self, raw_data, start):
        time.sleep(1)
        self.count = self.count + len(raw_data)
        fail_at = None
        if 'fail_at' in self.request.GET:
            fail_at = int(self.request.GET.get('fail_at'))
        if fail_at and self.count > fail_at:
            raise Exception('Triggered fail')

        return raw_data

    def file_complete(self, file_size):
        return None

########NEW FILE########
__FILENAME__ = advanced_settings
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from nose.tools import assert_false, assert_equal, assert_regexp_matches  # pylint: disable=E0611
from common import type_in_codemirror, press_the_notification_button, get_codemirror_value

KEY_CSS = '.key input.policy-key'
DISPLAY_NAME_KEY = "display_name"
DISPLAY_NAME_VALUE = '"Robot Super Course"'


@step('I select the Advanced Settings$')
def i_select_advanced_settings(step):

    world.click_course_settings()

    # The click handlers are set up so that if you click <body>
    # the menu disappears.  This means that if we're even a *little*
    # bit off on the last item ('Advanced Settings'), the menu
    # will close and the test will fail.
    # For this reason, we retrieve the link and visit it directly
    # This is what the browser *should* be doing, since it's just a native
    # link with no JavaScript involved.
    link_css = 'li.nav-course-settings-advanced a'
    world.wait_for_visible(link_css)
    link = world.css_find(link_css).first['href']
    world.visit(link)


@step('I am on the Advanced Course Settings page in Studio$')
def i_am_on_advanced_course_settings(step):
    step.given('I have opened a new course in Studio')
    step.given('I select the Advanced Settings')


@step(u'I edit the value of a policy key$')
def edit_the_value_of_a_policy_key(step):
    type_in_codemirror(get_index_of(DISPLAY_NAME_KEY), 'X')


@step(u'I edit the value of a policy key and save$')
def edit_the_value_of_a_policy_key_and_save(step):
    change_display_name_value(step, '"foo"')


@step('I create a JSON object as a value for "(.*)"$')
def create_JSON_object(step, key):
    change_value(step, key, '{"key": "value", "key_2": "value_2"}')


@step('I create a non-JSON value not in quotes$')
def create_value_not_in_quotes(step):
    change_display_name_value(step, 'quote me')


@step('I see default advanced settings$')
def i_see_default_advanced_settings(step):
    # Test only a few of the existing properties (there are around 34 of them)
    assert_policy_entries(
        ["advanced_modules", DISPLAY_NAME_KEY, "show_calculator"], ["[]", DISPLAY_NAME_VALUE, "false"])


@step('the settings are alphabetized$')
def they_are_alphabetized(step):
    key_elements = world.css_find(KEY_CSS)
    all_keys = []
    for key in key_elements:
        all_keys.append(key.value)

    assert_equal(sorted(all_keys), all_keys, "policy keys were not sorted")


@step('it is displayed as formatted$')
def it_is_formatted(step):
    assert_policy_entries(['discussion_topics'], ['{\n    "key": "value",\n    "key_2": "value_2"\n}'])


@step('I get an error on save$')
def error_on_save(step):
    assert_regexp_matches(world.css_text('#notification-error-description'), 'Incorrect setting format')


@step('it is displayed as a string')
def it_is_displayed_as_string(step):
    assert_policy_entries([DISPLAY_NAME_KEY], ['"quote me"'])


@step(u'the policy key value is unchanged$')
def the_policy_key_value_is_unchanged(step):
    assert_equal(get_display_name_value(), DISPLAY_NAME_VALUE)


@step(u'the policy key value is changed$')
def the_policy_key_value_is_changed(step):
    assert_equal(get_display_name_value(), '"foo"')


def assert_policy_entries(expected_keys, expected_values):
    for key, value in zip(expected_keys, expected_values):
        index = get_index_of(key)
        assert_false(index == -1, "Could not find key: {key}".format(key=key))
        found_value = get_codemirror_value(index)
        assert_equal(
            value, found_value,
            "Expected {} to have value {} but found {}".format(key, value, found_value)
        )


def get_index_of(expected_key):
    for i, element in enumerate(world.css_find(KEY_CSS)):
        # Sometimes get stale reference if I hold on to the array of elements
        key = world.css_value(KEY_CSS, index=i)
        if key == expected_key:
            return i

    return -1


def get_display_name_value():
    index = get_index_of(DISPLAY_NAME_KEY)
    return get_codemirror_value(index)

def change_display_name_value(step, new_value):
    change_value(step, DISPLAY_NAME_KEY, new_value)

def change_value(step, key, new_value):
    index = get_index_of(key)
    type_in_codemirror(index, new_value)
    press_the_notification_button(step, "Save")
    world.wait_for_ajax_complete()

########NEW FILE########
__FILENAME__ = checklists
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from nose.tools import assert_true, assert_equal  # pylint: disable=E0611
from terrain.steps import reload_the_page
from selenium.common.exceptions import StaleElementReferenceException


############### ACTIONS ####################
@step('I select Checklists from the Tools menu$')
def i_select_checklists(step):
    world.click_tools()
    link_css = 'li.nav-course-tools-checklists a'
    world.css_click(link_css)
    world.wait_for_ajax_complete()


@step('I have opened Checklists$')
def i_have_opened_checklists(step):
    step.given('I have opened a new course in Studio')
    step.given('I select Checklists from the Tools menu')


@step('I see the four default edX checklists$')
def i_see_default_checklists(step):
    checklists = world.css_find('.checklist-title')
    assert_equal(4, len(checklists))
    assert_true(checklists[0].text.endswith('Getting Started With Studio'))
    assert_true(checklists[1].text.endswith('Draft a Rough Course Outline'))
    assert_true(checklists[2].text.endswith("Explore edX\'s Support Tools"))
    assert_true(checklists[3].text.endswith('Draft Your Course About Page'))


@step('I can check and uncheck tasks in a checklist$')
def i_can_check_and_uncheck_tasks(step):
    # Use the 2nd checklist as a reference
    verifyChecklist2Status(0, 7, 0)
    toggleTask(1, 0)
    verifyChecklist2Status(1, 7, 14)
    toggleTask(1, 3)
    verifyChecklist2Status(2, 7, 29)
    toggleTask(1, 6)
    verifyChecklist2Status(3, 7, 43)
    toggleTask(1, 3)
    verifyChecklist2Status(2, 7, 29)


@step('the tasks are correctly selected$')
def tasks_correctly_selected(step):
    verifyChecklist2Status(2, 7, 29)
    # verify that task 7 is still selected by toggling its checkbox state and making sure that it deselects
    world.browser.execute_script("window.scrollBy(0,1000)")
    toggleTask(1, 6)
    verifyChecklist2Status(1, 7, 14)


@step('I select a link to the course outline$')
def i_select_a_link_to_the_course_outline(step):
    clickActionLink(1, 0, 'Edit Course Outline')


@step('I am brought to the course outline page$')
def i_am_brought_to_course_outline(step):
    assert world.is_css_present('body.view-outline')
    assert_equal(1, len(world.browser.windows))


@step('I am brought back to the course outline in the correct state$')
def i_am_brought_back_to_course_outline(step):
    step.given('I see the four default edX checklists')
    # In a previous step, we selected (1, 0) in order to click the 'Edit Course Outline' link.
    # Make sure the task is still showing as selected (there was a caching bug with the collection).
    verifyChecklist2Status(1, 7, 14)


@step('I select a link to help page$')
def i_select_a_link_to_the_help_page(step):
    clickActionLink(2, 0, 'Visit Studio Help')


@step('I am brought to the help page in a new window$')
def i_am_brought_to_help_page_in_new_window(step):
    step.given('I see the four default edX checklists')
    windows = world.browser.windows
    assert_equal(2, len(windows))
    world.browser.switch_to_window(windows[1])
    assert_equal('http://help.edge.edx.org/', world.browser.url)


############### HELPER METHODS ####################
def verifyChecklist2Status(completed, total, percentage):
    def verify_count(driver):
        try:
            statusCount = world.css_find('#course-checklist1 .status-count').first
            return statusCount.text == str(completed)
        except StaleElementReferenceException:
            return False

    world.wait_for(verify_count)
    assert_equal(str(total), world.css_find('#course-checklist1 .status-amount').first.text)
    # Would like to check the CSS width, but not sure how to do that.
    assert_equal(str(percentage), world.css_find('#course-checklist1 .viz-checklist-status-value .int').first.text)


def toggleTask(checklist, task):
    world.css_click('#course-checklist' + str(checklist) + '-task' + str(task))
    world.wait_for_ajax_complete()


# TODO: figure out a way to do this in phantom and firefox
# For now we will mark the scenerios that use this method as skipped
def clickActionLink(checklist, task, actionText):
    # text will be empty initially, wait for it to populate
    def verify_action_link_text(driver):
        actualText = world.css_text('#course-checklist' + str(checklist) + ' a', index=task)
        if actualText == actionText:
            return True
        else:
            # toggle checklist item to make sure that the link button is showing
            toggleTask(checklist, task)
            return False

    world.wait_for(verify_action_link_text)
    world.css_click('#course-checklist' + str(checklist) + ' a', index=task)
    world.wait_for_ajax_complete()

########NEW FILE########
__FILENAME__ = common
# pylint: disable=C0111
# pylint: disable=W0621

import os
from lettuce import world, step
from nose.tools import assert_true, assert_in  # pylint: disable=no-name-in-module
from django.conf import settings

from student.roles import CourseStaffRole, CourseInstructorRole, GlobalStaff
from student.models import get_user

from selenium.webdriver.common.keys import Keys

from logging import getLogger
from student.tests.factories import AdminFactory
from student import auth
logger = getLogger(__name__)

from terrain.browser import reset_data

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT


@step('I (?:visit|access|open) the Studio homepage$')
def i_visit_the_studio_homepage(_step):
    # To make this go to port 8001, put
    # LETTUCE_SERVER_PORT = 8001
    # in your settings.py file.
    world.visit('/')
    signin_css = 'a.action-signin'
    assert world.is_css_present(signin_css)


@step('I am logged into Studio$')
def i_am_logged_into_studio(_step):
    log_into_studio()


@step('I confirm the alert$')
def i_confirm_with_ok(_step):
    world.browser.get_alert().accept()


@step(u'I press the "([^"]*)" delete icon$')
def i_press_the_category_delete_icon(_step, category):
    if category == 'section':
        css = 'a.action.delete-section-button'
    elif category == 'subsection':
        css = 'a.action.delete-subsection-button'
    else:
        assert False, 'Invalid category: %s' % category
    world.css_click(css)


@step('I have opened a new course in Studio$')
def i_have_opened_a_new_course(_step):
    open_new_course()


@step('(I select|s?he selects) the new course')
def select_new_course(_step, whom):
    course_link_css = 'a.course-link'
    world.css_click(course_link_css)


@step(u'I press the "([^"]*)" notification button$')
def press_the_notification_button(_step, name):

    # Because the notification uses a CSS transition,
    # Selenium will always report it as being visible.
    # This makes it very difficult to successfully click
    # the "Save" button at the UI level.
    # Instead, we use JavaScript to reliably click
    # the button.
    btn_css = 'div#page-notification a.action-%s' % name.lower()
    world.trigger_event(btn_css, event='focus')
    world.browser.execute_script("$('{}').click()".format(btn_css))
    world.wait_for_ajax_complete()


@step('I change the "(.*)" field to "(.*)"$')
def i_change_field_to_value(_step, field, value):
    field_css = '#%s' % '-'.join([s.lower() for s in field.split()])
    ele = world.css_find(field_css).first
    ele.fill(value)
    ele._element.send_keys(Keys.ENTER)


@step('I reset the database')
def reset_the_db(_step):
    """
    When running Lettuce tests using examples (i.e. "Confirmation is
    shown on save" in course-settings.feature), the normal hooks
    aren't called between examples. reset_data should run before each
    scenario to flush the test database. When this doesn't happen we
    get errors due to trying to insert a non-unique entry. So instead,
    we delete the database manually. This has the effect of removing
    any users and courses that have been created during the test run.
    """
    reset_data(None)


@step('I see a confirmation that my changes have been saved')
def i_see_a_confirmation(step):
    confirmation_css = '#alert-confirmation'
    assert world.is_css_present(confirmation_css)


def open_new_course():
    world.clear_courses()
    create_studio_user()
    log_into_studio()
    create_a_course()


def create_studio_user(
        uname='robot',
        email='robot+studio@edx.org',
        password='test',
        is_staff=False):
    studio_user = world.UserFactory(
        username=uname,
        email=email,
        password=password,
        is_staff=is_staff)

    registration = world.RegistrationFactory(user=studio_user)
    registration.register(studio_user)
    registration.activate()

    return studio_user


def fill_in_course_info(
        name='Robot Super Course',
        org='MITx',
        num='101',
        run='2013_Spring'):
    world.css_fill('.new-course-name', name)
    world.css_fill('.new-course-org', org)
    world.css_fill('.new-course-number', num)
    world.css_fill('.new-course-run', run)


def log_into_studio(
        uname='robot',
        email='robot+studio@edx.org',
        password='test',
        name='Robot Studio'):

    world.log_in(username=uname, password=password, email=email, name=name)
    # Navigate to the studio dashboard
    world.visit('/')
    assert_in(uname, world.css_text('h2.title', timeout=10))


def add_course_author(user, course):
    """
    Add the user to the instructor group of the course
    so they will have the permissions to see it in studio
    """
    global_admin = AdminFactory()
    for role in (CourseStaffRole, CourseInstructorRole):
        auth.add_users(global_admin, role(course.id), user)


def create_a_course():
    course = world.CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
    world.scenario_dict['COURSE'] = course

    user = world.scenario_dict.get("USER")
    if not user:
        user = get_user('robot+studio@edx.org')

    add_course_author(user, course)

    # Navigate to the studio dashboard
    world.visit('/')
    course_link_css = 'a.course-link'
    world.css_click(course_link_css)
    course_title_css = 'span.course-title'
    assert_true(world.is_css_present(course_title_css))


def add_section(name='My Section'):
    link_css = 'a.new-courseware-section-button'
    world.css_click(link_css)
    name_css = 'input.new-section-name'
    save_css = 'input.new-section-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)
    span_css = 'span.section-name-span'
    assert_true(world.is_css_present(span_css))


def add_subsection(name='Subsection One'):
    css = 'a.new-subsection-item'
    world.css_click(css)
    name_css = 'input.new-subsection-name-input'
    save_css = 'input.new-subsection-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)


def set_date_and_time(date_css, desired_date, time_css, desired_time, key=None):
    set_element_value(date_css, desired_date, key)
    world.wait_for_ajax_complete()

    set_element_value(time_css, desired_time, key)
    world.wait_for_ajax_complete()


def set_element_value(element_css, element_value, key=None):
    element = world.css_find(element_css).first
    element.fill(element_value)
    # hit TAB or provided key to trigger save content
    if key is not None:
        element._element.send_keys(getattr(Keys, key))  # pylint: disable=protected-access
    else:
        element._element.send_keys(Keys.TAB)  # pylint: disable=protected-access


@step('I have enabled the (.*) advanced module$')
def i_enabled_the_advanced_module(step, module):
    step.given('I have opened a new course section in Studio')
    world.css_click('.nav-course-settings')
    world.css_click('.nav-course-settings-advanced a')
    type_in_codemirror(0, '["%s"]' % module)
    press_the_notification_button(step, 'Save')


@world.absorb
def create_course_with_unit():
    """
    Prepare for tests by creating a course with a section, subsection, and unit.
    Performs the following:
        Clear out all courseware
        Create a course with a section, subsection, and unit
        Create a user and make that user a course author
        Log the user into studio
        Open the course from the dashboard
        Expand the section and click on the New Unit link
    The end result is the page where the user is editing the new unit
    """
    world.clear_courses()
    course = world.CourseFactory.create()
    world.scenario_dict['COURSE'] = course
    section = world.ItemFactory.create(parent_location=course.location)
    world.ItemFactory.create(
        parent_location=section.location,
        category='sequential',
        display_name='Subsection One',
    )
    user = create_studio_user(is_staff=False)
    add_course_author(user, course)

    log_into_studio()
    world.css_click('a.course-link')

    world.wait_for_js_to_load()
    css_selectors = [
        'div.section-item a.expand-collapse', 'a.new-unit-item'
    ]
    for selector in css_selectors:
        world.css_click(selector)

    world.wait_for_mathjax()
    world.wait_for_xmodule()

    assert world.is_css_present('ul.new-component-type')


@step('I have clicked the new unit button$')
@step(u'I am in Studio editing a new unit$')
def edit_new_unit(step):
    create_course_with_unit()


@step('the save notification button is disabled')
def save_button_disabled(step):
    button_css = '.action-save'
    disabled = 'is-disabled'
    assert world.css_has_class(button_css, disabled)


@step('the "([^"]*)" button is disabled')
def button_disabled(step, value):
    button_css = 'input[value="%s"]' % value
    assert world.css_has_class(button_css, 'is-disabled')


def _do_studio_prompt_action(intent, action):
    """
    Wait for a studio prompt to appear and press the specified action button
    See cms/static/js/views/feedback_prompt.js for implementation
    """
    assert intent in ['warning', 'error', 'confirmation', 'announcement',
        'step-required', 'help', 'mini']
    assert action in ['primary', 'secondary']

    world.wait_for_present('div.wrapper-prompt.is-shown#prompt-{}'.format(intent))

    action_css = 'li.nav-item > a.action-{}'.format(action)
    world.trigger_event(action_css, event='focus')
    world.browser.execute_script("$('{}').click()".format(action_css))

    world.wait_for_ajax_complete()
    world.wait_for_present('div.wrapper-prompt.is-hiding#prompt-{}'.format(intent))


@world.absorb
def confirm_studio_prompt():
    _do_studio_prompt_action('warning', 'primary')


@step('I confirm the prompt')
def confirm_the_prompt(step):
    confirm_studio_prompt()


@step(u'I am shown a prompt$')
def i_am_shown_a_notification(step):
    assert world.is_css_present('.wrapper-prompt')


def type_in_codemirror(index, text, find_prefix="$"):
    script = """
    var cm = {find_prefix}('div.CodeMirror:eq({index})').get(0).CodeMirror;
    cm.getInputField().focus();
    cm.setValue(arguments[0]);
    cm.getInputField().blur();""".format(index=index, find_prefix=find_prefix)
    world.browser.driver.execute_script(script, str(text))
    world.wait_for_ajax_complete()


def get_codemirror_value(index=0, find_prefix="$"):
    return world.browser.driver.execute_script(
        """
        return {find_prefix}('div.CodeMirror:eq({index})').get(0).CodeMirror.getValue();
        """.format(index=index, find_prefix=find_prefix)
    )



def attach_file(filename, sub_path):
    path = os.path.join(TEST_ROOT, sub_path, filename)
    world.browser.execute_script("$('input.file-input').css('display', 'block')")
    world.browser.attach_file('file', os.path.abspath(path))


def upload_file(filename, sub_path=''):
    attach_file(filename, sub_path)
    button_css = '.wrapper-modal-window-assetupload .action-upload'
    world.css_click(button_css)


@step(u'"([^"]*)" logs in$')
def other_user_login(step, name):
    step.given('I log out')
    world.visit('/')

    signin_css = 'a.action-signin'
    world.is_css_present(signin_css)
    world.css_click(signin_css)

    def fill_login_form():
        login_form = world.browser.find_by_css('form#login_form')
        login_form.find_by_name('email').fill(name + '@edx.org')
        login_form.find_by_name('password').fill("test")
        login_form.find_by_name('submit').click()
    world.retry_on_exception(fill_login_form)
    assert_true(world.is_css_present('.new-course-button'))
    world.scenario_dict['USER'] = get_user(name + '@edx.org')


@step(u'the user "([^"]*)" exists( as a course (admin|staff member|is_staff))?$')
def create_other_user(_step, name, has_extra_perms, role_name):
    email = name + '@edx.org'
    user = create_studio_user(uname=name, password="test", email=email)
    if has_extra_perms:
        if role_name == "is_staff":
            GlobalStaff().add_users(user)
        else:
            if role_name == "admin":
                # admins get staff privileges, as well
                roles = (CourseStaffRole, CourseInstructorRole)
            else:
                roles = (CourseStaffRole,)
            course_key = world.scenario_dict["COURSE"].id
            global_admin = AdminFactory()
            for role in roles:
                auth.add_users(global_admin, role(course_key), user)


@step('I log out')
def log_out(_step):
    world.visit('logout')


@step(u'I click on "edit a draft"$')
def i_edit_a_draft(_step):
    world.css_click("a.create-draft")


@step(u'I click on "replace with draft"$')
def i_replace_w_draft(_step):
    world.css_click("a.publish-draft")


@step(u'I click on "delete draft"$')
def i_delete_draft(_step):
    world.css_click("a.delete-draft")


@step(u'I publish the unit$')
def publish_unit(_step):
    world.select_option('visibility-select', 'public')


@step(u'I unpublish the unit$')
def unpublish_unit(_step):
    world.select_option('visibility-select', 'private')

########NEW FILE########
__FILENAME__ = component
# pylint: disable=C0111
# pylint: disable=W0621

# Lettuce formats proposed definitions for unimplemented steps with the
# argument name "step" instead of "_step" and pylint does not like that.
# pylint: disable=W0613

from lettuce import world, step
from nose.tools import assert_true, assert_in  # pylint: disable=E0611

DISPLAY_NAME = "Display Name"


@step(u'I add this type of single step component:$')
def add_a_single_step_component(step):
    for step_hash in step.hashes:
        component = step_hash['Component']
        assert_in(component, ['Discussion', 'Video'])

        world.create_component_instance(
            step=step,
            category='{}'.format(component.lower()),
        )


@step(u'I see this type of single step component:$')
def see_a_single_step_component(step):
    for step_hash in step.hashes:
        component = step_hash['Component']
        assert_in(component, ['Discussion', 'Video'])
        component_css = 'div.xmodule_{}Module'.format(component)
        assert_true(world.is_css_present(component_css),
                    "{} couldn't be found".format(component))


@step(u'I add this type of( Advanced)? (HTML|Problem) component:$')
def add_a_multi_step_component(step, is_advanced, category):
    for step_hash in step.hashes:
        world.create_component_instance(
            step=step,
            category='{}'.format(category.lower()),
            component_type=step_hash['Component'],
            is_advanced=bool(is_advanced),
        )


@step(u'I see (HTML|Problem) components in this order:')
def see_a_multi_step_component(step, category):

    # Wait for all components to finish rendering
    selector = 'li.component div.xblock-student_view'
    world.wait_for(lambda _: len(world.css_find(selector)) == len(step.hashes))

    for idx, step_hash in enumerate(step.hashes):

        if category == 'HTML':
            html_matcher = {
                'Text':
                    '\n    \n',
                'Announcement':
                    '<p> Words of encouragement! This is a short note that most students will read. </p>',
                'Zooming Image':
                    '<h2>ZOOMING DIAGRAMS</h2>',
                'E-text Written in LaTeX':
                    '<h2>Example: E-text page</h2>',
                'Raw HTML':
                    '<p>This template is similar to the Text template. The only difference is',
            }
            actual_html = world.css_html(selector, index=idx)
            assert_in(html_matcher[step_hash['Component']], actual_html)
        else:
            actual_text = world.css_text(selector, index=idx)
            assert_in(step_hash['Component'].upper(), actual_text)


@step(u'I see a "([^"]*)" Problem component$')
def see_a_problem_component(step, category):
    component_css = 'div.xmodule_CapaModule'
    assert_true(world.is_css_present(component_css),
                'No problem was added to the unit.')

    problem_css = 'li.component div.xblock-student_view'
    actual_text = world.css_text(problem_css)
    assert_in(category.upper(), actual_text)


@step(u'I add a "([^"]*)" "([^"]*)" component$')
def add_component_category(step, component, category):
    assert category in ('single step', 'HTML', 'Problem', 'Advanced Problem')
    given_string = 'I add this type of {} component:'.format(category)
    step.given('{}\n{}\n{}'.format(given_string, '|Component|', '|{}|'.format(component)))


@step(u'I delete all components$')
def delete_all_components(step):
    count = len(world.css_find('ol.components li.component'))
    step.given('I delete "' + str(count) + '" component')


@step(u'I delete "([^"]*)" component$')
def delete_components(step, number):
    world.wait_for_xmodule()
    delete_btn_css = 'a.delete-button'
    prompt_css = 'div#prompt-warning'
    btn_css = '{} a.button.action-primary'.format(prompt_css)
    saving_mini_css = 'div#page-notification .wrapper-notification-mini'
    for _ in range(int(number)):
        world.css_click(delete_btn_css)
        assert_true(
            world.is_css_present('{}.is-shown'.format(prompt_css)),
            msg='Waiting for the confirmation prompt to be shown')

        # Pressing the button via css was not working reliably for the last component
        # when run in Chrome.
        if world.browser.driver_name is 'Chrome':
            world.browser.execute_script("$('{}').click()".format(btn_css))
        else:
            world.css_click(btn_css)

        # Wait for the saving notification to pop up then disappear
        if world.is_css_present('{}.is-shown'.format(saving_mini_css)):
            world.css_find('{}.is-hiding'.format(saving_mini_css))


@step(u'I see no components')
def see_no_components(steps):
    assert world.is_css_not_present('li.component')


@step(u'I delete a component')
def delete_one_component(step):
    world.css_click('a.delete-button')


@step(u'I edit and save a component')
def edit_and_save_component(step):
    world.css_click('.edit-button')
    world.css_click('.save-button')


@step(u'I duplicate the (first|second|third) component$')
def duplicated_component(step, ordinal):
    ord_map = {
        "first": 0,
        "second": 1,
        "third": 2,
    }
    index = ord_map[ordinal]
    duplicate_btn_css = 'a.duplicate-button'
    world.css_click(duplicate_btn_css, int(index))


@step(u'I see a Problem component with display name "([^"]*)" in position "([^"]*)"$')
def see_component_in_position(step, display_name, index):
    component_css = 'div.xmodule_CapaModule'

    def find_problem(_driver):
        return world.css_text(component_css, int(index)).startswith(display_name.upper())

    world.wait_for(find_problem, timeout_msg='Did not find the duplicated problem')


@step(u'I see the display name is "([^"]*)"')
def check_component_display_name(step, display_name):
    label = world.css_text(".component-header")
    assert display_name == label


@step(u'I change the display name to "([^"]*)"')
def change_display_name(step, display_name):
    world.edit_component_and_select_settings()
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, display_name)
    world.save_component()


@step(u'I unset the display name')
def unset_display_name(step):
    world.edit_component_and_select_settings()
    world.revert_setting_entry(DISPLAY_NAME)
    world.save_component()

########NEW FILE########
__FILENAME__ = component_settings_editor_helpers
# disable missing docstring
# pylint: disable=C0111

from lettuce import world
from nose.tools import assert_equal, assert_in  # pylint: disable=E0611
from terrain.steps import reload_the_page
from common import type_in_codemirror
from selenium.webdriver.common.keys import Keys
from cms.envs.common import FEATURES


@world.absorb
def create_component_instance(step, category, component_type=None, is_advanced=False):
    """
    Create a new component in a Unit.

    Parameters
    ----------
    category: component type (discussion, html, problem, video)
    component_type: for components with multiple templates, the link text in the menu
    is_advanced: for problems, is the desired component under the advanced menu?
    """
    assert_in(category, ['problem', 'html', 'video', 'discussion'])

    component_button_css = 'span.large-{}-icon'.format(category.lower())
    if category == 'problem':
        module_css = 'div.xmodule_CapaModule'
    else:
        module_css = 'div.xmodule_{}Module'.format(category.title())

    # Count how many of that module is on the page. Later we will
    # assert that one more was added.
    # We need to use world.browser.find_by_css instead of world.css_find
    # because it's ok if there are currently zero of them.
    module_count_before =  len(world.browser.find_by_css(module_css))

    # Disable the jquery animation for the transition to the menus.
    world.disable_jquery_animations()
    world.css_click(component_button_css)

    if category in ('problem', 'html'):
        world.wait_for_invisible(component_button_css)
        click_component_from_menu(category, component_type, is_advanced)

    expected_count = module_count_before + 1
    world.wait_for(
        lambda _: len(world.css_find(module_css)) == expected_count,
        timeout=20
    )


@world.absorb
def click_new_component_button(step, component_button_css):
    step.given('I have clicked the new unit button')

    world.css_click(component_button_css)


def _click_advanced():
    css = 'ul.problem-type-tabs a[href="#tab2"]'
    world.css_click(css)

    # Wait for the advanced tab items to be displayed
    tab2_css = 'div.ui-tabs-panel#tab2'
    world.wait_for_visible(tab2_css)


def _find_matching_link(category, component_type):
    """
    Find the link with the specified text. There should be one and only one.
    """

    # The tab shows links for the given category
    links = world.css_find('div.new-component-{} a'.format(category))

    # Find the link whose text matches what you're looking for
    matched_links = [link for link in links if link.text == component_type]

    # There should be one and only one
    assert_equal(len(matched_links), 1)
    return matched_links[0]


def click_component_from_menu(category, component_type, is_advanced):
    """
    Creates a component for a category with more
    than one template, i.e. HTML and Problem.
    For some problem types, it is necessary to click to
    the Advanced tab.
    The component_type is the link text, e.g. "Blank Common Problem"
    """
    if is_advanced:
        # Sometimes this click does not work if you go too fast.
        world.retry_on_exception(_click_advanced,
            ignored_exceptions=AssertionError)

    # Retry this in case the list is empty because you tried too fast.
    link = world.retry_on_exception(
        lambda: _find_matching_link(category, component_type),
        ignored_exceptions=AssertionError
    )

    # Wait for the link to be clickable. If you go too fast it is not.
    world.retry_on_exception(lambda: link.click())


@world.absorb
def edit_component_and_select_settings():
    world.edit_component()
    world.ensure_settings_visible()


@world.absorb
def ensure_settings_visible():
    # Select the 'settings' tab if there is one (it isn't displayed if it is the only option)
    settings_button = world.browser.find_by_css('.settings-button')
    if len(settings_button) > 0:
        world.css_click('.settings-button')


@world.absorb
def edit_component():
    world.wait_for(lambda _driver: world.css_visible('a.edit-button'))
    world.css_click('a.edit-button')
    world.wait_for_ajax_complete()


@world.absorb
def select_editor_tab(tab_name):
    editor_tabs = world.browser.find_by_css('.editor-tabs a')
    expected_tab_text = tab_name.strip().upper()
    matching_tabs = [tab for tab in editor_tabs if tab.text.upper() == expected_tab_text]
    assert len(matching_tabs) == 1
    tab = matching_tabs[0]
    tab.click()
    world.wait_for_ajax_complete()


def enter_xml_in_advanced_problem(step, text):
    """
    Edits an advanced problem (assumes only on page),
    types the provided XML, and saves the component.
    """
    world.edit_component()
    type_in_codemirror(0, text)
    world.save_component()


@world.absorb
def verify_setting_entry(setting, display_name, value, explicitly_set):
    """
    Verify the capa module fields are set as expected in the
    Advanced Settings editor.

    Parameters
    ----------
    setting: the WebDriverElement object found in the browser
    display_name: the string expected as the label
    html: the expected field value
    explicitly_set: True if the value is expected to have been explicitly set
        for the problem, rather than derived from the defaults. This is verified
        by the existence of a "Clear" button next to the field value.
    """
    assert_equal(display_name, setting.find_by_css('.setting-label')[0].html.strip())

    # Check if the web object is a list type
    # If so, we use a slightly different mechanism for determining its value
    if setting.has_class('metadata-list-enum') or setting.has_class('metadata-dict') or setting.has_class('metadata-video-translations'):
        list_value = ', '.join(ele.value for ele in setting.find_by_css('.list-settings-item'))
        assert_equal(value, list_value)
    elif setting.has_class('metadata-videolist-enum'):
        list_value = ', '.join(ele.find_by_css('input')[0].value for ele in setting.find_by_css('.videolist-settings-item'))
        assert_equal(value, list_value)
    else:
        assert_equal(value, setting.find_by_css('.setting-input')[0].value)

    # VideoList doesn't have clear button
    if not setting.has_class('metadata-videolist-enum'):
        settingClearButton = setting.find_by_css('.setting-clear')[0]
        assert_equal(explicitly_set, settingClearButton.has_class('active'))
        assert_equal(not explicitly_set, settingClearButton.has_class('inactive'))


@world.absorb
def verify_all_setting_entries(expected_entries):
    settings = world.browser.find_by_css('.wrapper-comp-setting')
    assert_equal(len(expected_entries), len(settings))
    for (counter, setting) in enumerate(settings):
        world.verify_setting_entry(
            setting, expected_entries[counter][0],
            expected_entries[counter][1], expected_entries[counter][2]
        )


@world.absorb
def save_component():
    world.css_click("a.action-save")
    world.wait_for_ajax_complete()


@world.absorb
def save_component_and_reopen(step):
    save_component()
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes WERE persisted after Save.
    reload_the_page(step)
    edit_component_and_select_settings()


@world.absorb
def cancel_component(step):
    world.css_click("a.action-cancel")
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes were not persisted.
    reload_the_page(step)


@world.absorb
def revert_setting_entry(label):
    get_setting_entry(label).find_by_css('.setting-clear')[0].click()


@world.absorb
def get_setting_entry(label):
    def get_setting():
        settings = world.css_find('.wrapper-comp-setting')
        for setting in settings:
            if setting.find_by_css('.setting-label')[0].value == label:
                return setting
        return None
    return world.retry_on_exception(get_setting)


@world.absorb
def get_setting_entry_index(label):
    def get_index():
        settings = world.css_find('.metadata_edit .wrapper-comp-setting')
        for index, setting in enumerate(settings):
            if setting.find_by_css('.setting-label')[0].value == label:
                return index
        return None
    return world.retry_on_exception(get_index)


@world.absorb
def set_field_value(index, value):
    """
    Set the field to the specified value.

    Note: we cannot use css_fill here because the value is not set
    until after you move away from that field.
    Instead we will find the element, set its value, then hit the Tab key
    to get to the next field.
    """
    elem = world.css_find('.metadata_edit div.wrapper-comp-setting input.setting-input')[index]
    elem.value = value
    elem.type(Keys.TAB)

########NEW FILE########
__FILENAME__ = course-export
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

from lettuce import world, step
from component_settings_editor_helpers import enter_xml_in_advanced_problem
from nose.tools import assert_true, assert_equal
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_usage_url


@step('I go to the export page$')
def i_go_to_the_export_page(step):
    world.click_tools()
    link_css = 'li.nav-course-tools-export a'
    world.css_click(link_css)


@step('I export the course$')
def i_export_the_course(step):
    step.given('I go to the export page')
    world.css_click('a.action-export')


@step('I edit and enter bad XML$')
def i_enter_bad_xml(step):
    enter_xml_in_advanced_problem(step,
        """<problem><h1>Smallest Canvas</h1>
            <p>You want to make the smallest canvas you can.</p>
            <multiplechoiceresponse>
            <choicegroup type="MultipleChoice">
              <choice correct="false"><verbatim><canvas id="myCanvas" width = 10 height = 100> </canvas></verbatim></choice>
              <choice correct="true"><code><canvas id="myCanvas" width = 10 height = 10> </canvas></code></choice>
            </choicegroup>
            </multiplechoiceresponse>
            </problem>"""
    )


@step('I edit and enter an ampersand$')
def i_enter_an_ampersand(step):
    enter_xml_in_advanced_problem(step, "<problem>&</problem>")


@step('I get an error dialog$')
def get_an_error_dialog(step):
    assert_true(world.is_css_present("div.prompt.error"))


@step('I can click to go to the unit with the error$')
def i_click_on_error_dialog(step):
    world.click_link_by_text('Correct failed component')
    assert_true(world.css_html("span.inline-error").startswith("Problem i4x://MITx/999/problem"))
    course_key = SlashSeparatedCourseKey("MITx", "999", "Robot_Super_Course")
    # we don't know the actual ID of the vertical. So just check that we did go to a
    # vertical page in the course (there should only be one).
    vertical_usage_key = course_key.make_usage_key("vertical", "")
    vertical_url = reverse_usage_url('unit_handler', vertical_usage_key)
    assert_equal(1, world.browser.url.count(vertical_url))

########NEW FILE########
__FILENAME__ = course-overview
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from nose.tools import assert_true, assert_false, assert_equal  # pylint: disable=E0611

from logging import getLogger
logger = getLogger(__name__)


@step(u'I have a course with no sections$')
def have_a_course(step):
    world.clear_courses()
    course = world.CourseFactory.create()


@step(u'I have a course with 1 section$')
def have_a_course_with_1_section(step):
    world.clear_courses()
    course = world.CourseFactory.create()
    section = world.ItemFactory.create(parent_location=course.location)
    subsection1 = world.ItemFactory.create(
        parent_location=section.location,
        category='sequential',
        display_name='Subsection One',)


@step(u'I have a course with multiple sections$')
def have_a_course_with_two_sections(step):
    world.clear_courses()
    course = world.CourseFactory.create()
    section = world.ItemFactory.create(parent_location=course.location)
    subsection1 = world.ItemFactory.create(
        parent_location=section.location,
        category='sequential',
        display_name='Subsection One',)
    section2 = world.ItemFactory.create(
        parent_location=course.location,
        display_name='Section Two',)
    subsection2 = world.ItemFactory.create(
        parent_location=section2.location,
        category='sequential',
        display_name='Subsection Alpha',)
    subsection3 = world.ItemFactory.create(
        parent_location=section2.location,
        category='sequential',
        display_name='Subsection Beta',)


@step(u'I navigate to the course overview page$')
def navigate_to_the_course_overview_page(step):
    create_studio_user(is_staff=True)
    log_into_studio()
    course_locator = 'a.course-link'
    world.css_click(course_locator)


@step(u'I navigate to the courseware page of a course with multiple sections')
def nav_to_the_courseware_page_of_a_course_with_multiple_sections(step):
    step.given('I have a course with multiple sections')
    step.given('I navigate to the course overview page')


@step(u'I add a section')
def i_add_a_section(step):
    add_section(name='My New Section That I Just Added')


@step(u'I click the "([^"]*)" link$')
def i_click_the_text_span(step, text):
    span_locator = '.toggle-button-sections span'
    assert_true(world.browser.is_element_present_by_css(span_locator))
    # first make sure that the expand/collapse text is the one you expected
    assert_true(world.css_has_value(span_locator, text))
    world.css_click(span_locator)


@step(u'I collapse the first section$')
def i_collapse_a_section(step):
    collapse_locator = 'section.courseware-section a.collapse'
    world.css_click(collapse_locator)


@step(u'I expand the first section$')
def i_expand_a_section(step):
    expand_locator = 'section.courseware-section a.expand'
    world.css_click(expand_locator)


@step(u'I see the "([^"]*)" link$')
def i_see_the_span_with_text(step, text):
    span_locator = '.toggle-button-sections span'
    assert_true(world.css_has_value(span_locator, text))
    assert_true(world.css_visible(span_locator))


@step(u'I do not see the "([^"]*)" link$')
def i_do_not_see_the_span_with_text(step, text):
    # Note that the span will exist on the page but not be visible
    span_locator = '.toggle-button-sections span'
    assert_true(world.is_css_present(span_locator))
    assert_false(world.css_visible(span_locator))


@step(u'all sections are expanded$')
def all_sections_are_expanded(step):
    subsection_locator = 'div.subsection-list'
    subsections = world.css_find(subsection_locator)
    for index in range(len(subsections)):
        assert_true(world.css_visible(subsection_locator, index=index))


@step(u'all sections are collapsed$')
def all_sections_are_collapsed(step):
    subsection_locator = 'div.subsection-list'
    subsections = world.css_find(subsection_locator)
    for index in range(len(subsections)):
        assert_false(world.css_visible(subsection_locator, index=index))


@step(u"I change an assignment's grading status")
def change_grading_status(step):
    world.css_find('a.menu-toggle').click()
    world.css_find('.menu li').first.click()


@step(u'I reorder subsections')
def reorder_subsections(_step):
    draggable_css = '.subsection-drag-handle'
    ele = world.css_find(draggable_css).first
    ele.action_chains.drag_and_drop_by_offset(
        ele._element,
        0,
        25
    ).perform()

########NEW FILE########
__FILENAME__ = course-settings
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from terrain.steps import reload_the_page
from selenium.webdriver.common.keys import Keys
from common import type_in_codemirror, upload_file
from django.conf import settings

from nose.tools import assert_true, assert_false, assert_equal  # pylint: disable=E0611

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT

COURSE_START_DATE_CSS = "#course-start-date"
COURSE_END_DATE_CSS = "#course-end-date"
ENROLLMENT_START_DATE_CSS = "#course-enrollment-start-date"
ENROLLMENT_END_DATE_CSS = "#course-enrollment-end-date"

COURSE_START_TIME_CSS = "#course-start-time"
COURSE_END_TIME_CSS = "#course-end-time"
ENROLLMENT_START_TIME_CSS = "#course-enrollment-start-time"
ENROLLMENT_END_TIME_CSS = "#course-enrollment-end-time"

DUMMY_TIME = "15:30"
DEFAULT_TIME = "00:00"


############### ACTIONS ####################
@step('I select Schedule and Details$')
def test_i_select_schedule_and_details(step):
    world.click_course_settings()
    link_css = 'li.nav-course-settings-schedule a'
    world.css_click(link_css)
    world.wait_for_requirejs(
        ["jquery", "js/models/course",
         "js/models/settings/course_details", "js/views/settings/main"])


@step('I have set course dates$')
def test_i_have_set_course_dates(step):
    step.given('I have opened a new course in Studio')
    step.given('I select Schedule and Details')
    step.given('And I set course dates')


@step('And I set course dates$')
def test_and_i_set_course_dates(step):
    set_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    set_date_or_time(COURSE_END_DATE_CSS, '12/26/2013')
    set_date_or_time(ENROLLMENT_START_DATE_CSS, '12/1/2013')
    set_date_or_time(ENROLLMENT_END_DATE_CSS, '12/10/2013')

    set_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)
    set_date_or_time(ENROLLMENT_END_TIME_CSS, DUMMY_TIME)


@step('And I clear all the dates except start$')
def test_and_i_clear_all_the_dates_except_start(step):
    set_date_or_time(COURSE_END_DATE_CSS, '')
    set_date_or_time(ENROLLMENT_START_DATE_CSS, '')
    set_date_or_time(ENROLLMENT_END_DATE_CSS, '')


@step('Then I see cleared dates$')
def test_then_i_see_cleared_dates(step):
    verify_date_or_time(COURSE_END_DATE_CSS, '')
    verify_date_or_time(ENROLLMENT_START_DATE_CSS, '')
    verify_date_or_time(ENROLLMENT_END_DATE_CSS, '')

    verify_date_or_time(COURSE_END_TIME_CSS, '')
    verify_date_or_time(ENROLLMENT_START_TIME_CSS, '')
    verify_date_or_time(ENROLLMENT_END_TIME_CSS, '')

    # Verify course start date (required) and time still there
    verify_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)


@step('I clear the course start date$')
def test_i_clear_the_course_start_date(step):
    set_date_or_time(COURSE_START_DATE_CSS, '')


@step('I receive a warning about course start date$')
def test_i_receive_a_warning_about_course_start_date(step):
    assert_true(world.css_has_text('.message-error', 'The course must have an assigned start date.'))
    assert_true('error' in world.css_find(COURSE_START_DATE_CSS).first._element.get_attribute('class'))
    assert_true('error' in world.css_find(COURSE_START_TIME_CSS).first._element.get_attribute('class'))


@step('the previously set start date is shown$')
def test_the_previously_set_start_date_is_shown(step):
    verify_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)


@step('Given I have tried to clear the course start$')
def test_i_have_tried_to_clear_the_course_start(step):
    step.given("I have set course dates")
    step.given("I clear the course start date")
    step.given("I receive a warning about course start date")


@step('I have entered a new course start date$')
def test_i_have_entered_a_new_course_start_date(step):
    set_date_or_time(COURSE_START_DATE_CSS, '12/22/2013')


@step('The warning about course start date goes away$')
def test_the_warning_about_course_start_date_goes_away(step):
    assert world.is_css_not_present('.message-error')
    assert_false('error' in world.css_find(COURSE_START_DATE_CSS).first._element.get_attribute('class'))
    assert_false('error' in world.css_find(COURSE_START_TIME_CSS).first._element.get_attribute('class'))


@step('my new course start date is shown$')
def new_course_start_date_is_shown(step):
    verify_date_or_time(COURSE_START_DATE_CSS, '12/22/2013')
    # Time should have stayed from before attempt to clear date.
    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)


@step('I change fields$')
def test_i_change_fields(step):
    set_date_or_time(COURSE_START_DATE_CSS, '7/7/7777')
    set_date_or_time(COURSE_END_DATE_CSS, '7/7/7777')
    set_date_or_time(ENROLLMENT_START_DATE_CSS, '7/7/7777')
    set_date_or_time(ENROLLMENT_END_DATE_CSS, '7/7/7777')


@step('I change the course overview')
def test_change_course_overview(_step):
    type_in_codemirror(0, "<h1>Overview</h1>")


@step('I click the "Upload Course Image" button')
def click_upload_button(_step):
    button_css = '.action-upload-image'
    world.css_click(button_css)


@step('I upload a new course image$')
def upload_new_course_image(_step):
    upload_file('image.jpg')


@step('I should see the new course image$')
def i_see_new_course_image(_step):
    img_css = '#course-image'
    images = world.css_find(img_css)
    assert len(images) == 1
    img = images[0]
    expected_src = '/c4x/MITx/999/asset/image.jpg'

    # Don't worry about the domain in the URL
    success_func = lambda _: img['src'].endswith(expected_src)
    world.wait_for(success_func)


@step('the image URL should be present in the field')
def image_url_present(_step):
    field_css = '#course-image-url'
    expected_value = '/c4x/MITx/999/asset/image.jpg'
    assert world.css_value(field_css) == expected_value


############### HELPER METHODS ####################
def set_date_or_time(css, date_or_time):
    """
    Sets date or time field.
    """
    world.css_fill(css, date_or_time)
    e = world.css_find(css).first
    # hit Enter to apply the changes
    e._element.send_keys(Keys.ENTER)


def verify_date_or_time(css, date_or_time):
    """
    Verifies date or time field.
    """
    # We need to wait for JavaScript to fill in the field, so we use
    # css_has_value(), which first checks that the field is not blank
    assert_true(world.css_has_value(css, date_or_time))


@step('I do not see the changes')
@step('I see the set dates')
def i_see_the_set_dates(_step):
    """
    Ensure that each field has the value set in `test_and_i_set_course_dates`.
    """
    verify_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    verify_date_or_time(COURSE_END_DATE_CSS, '12/26/2013')
    verify_date_or_time(ENROLLMENT_START_DATE_CSS, '12/01/2013')
    verify_date_or_time(ENROLLMENT_END_DATE_CSS, '12/10/2013')

    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)
    # Unset times get set to 12 AM once the corresponding date has been set.
    verify_date_or_time(COURSE_END_TIME_CSS, DEFAULT_TIME)
    verify_date_or_time(ENROLLMENT_START_TIME_CSS, DEFAULT_TIME)
    verify_date_or_time(ENROLLMENT_END_TIME_CSS, DUMMY_TIME)

########NEW FILE########
__FILENAME__ = course-team
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from nose.tools import assert_in  # pylint: disable=E0611


@step(u'(I am viewing|s?he views) the course team settings$')
def view_grading_settings(_step, whom):
    world.click_course_settings()
    link_css = 'li.nav-course-settings-team a'
    world.css_click(link_css)


@step(u'I add "([^"]*)" to the course team$')
def add_other_user(_step, name):
    new_user_css = 'a.create-user-button'
    world.css_click(new_user_css)

    # Wait for the css animation to apply the is-shown class
    shown_css = 'div.wrapper-create-user.is-shown'
    world.wait_for_present(shown_css)

    email_css = 'input#user-email-input'
    world.css_fill(email_css, name + '@edx.org')
    if world.is_firefox():
        world.trigger_event(email_css)
    confirm_css = 'form.create-user button.action-primary'
    world.css_click(confirm_css)


@step(u'I delete "([^"]*)" from the course team$')
def delete_other_user(_step, name):
    to_delete_css = '.user-item .item-actions a.remove-user[data-id="{email}"]'.format(
        email="{0}{1}".format(name, '@edx.org'))
    world.css_click(to_delete_css)
    world.confirm_studio_prompt()


@step(u's?he deletes me from the course team$')
def other_delete_self(_step):
    to_delete_css = '.user-item .item-actions a.remove-user[data-id="{email}"]'.format(
        email="robot+studio@edx.org")
    world.css_click(to_delete_css)
    world.confirm_studio_prompt()


@step(u'I make "([^"]*)" a course team admin$')
def make_course_team_admin(_step, name):
    admin_btn_css = '.user-item[data-email="{name}@edx.org"] .user-actions .add-admin-role'.format(
        name=name)
    world.css_click(admin_btn_css)


@step(u'I remove admin rights from ("([^"]*)"|myself)$')
def remove_course_team_admin(_step, outer_capture, name):
    if outer_capture == "myself":
        email = world.scenario_dict["USER"].email
    else:
        email = name + '@edx.org'
    admin_btn_css = '.user-item[data-email="{email}"] .user-actions .remove-admin-role'.format(
        email=email)
    world.css_click(admin_btn_css)


@step(u'I( do not)? see the course on my page$')
@step(u's?he does( not)? see the course on (his|her) page$')
def see_course(_step, do_not_see, gender='self'):
    class_css = 'h3.course-title'
    if do_not_see:
        assert world.is_css_not_present(class_css)
    else:
        all_courses = world.css_find(class_css)
        all_names = [item.html for item in all_courses]
        assert_in(world.scenario_dict['COURSE'].display_name, all_names)


@step(u'"([^"]*)" should( not)? be marked as an admin$')
def marked_as_admin(_step, name, not_marked_admin):
    flag_css = '.user-item[data-email="{name}@edx.org"] .flag-role.flag-role-admin'.format(
        name=name)
    if not_marked_admin:
        assert world.is_css_not_present(flag_css)
    else:
        assert world.is_css_present(flag_css)


@step(u'I should( not)? be marked as an admin$')
def self_marked_as_admin(_step, not_marked_admin):
    return marked_as_admin(_step, "robot+studio", not_marked_admin)


@step(u'I can(not)? delete users$')
@step(u's?he can(not)? delete users$')
def can_delete_users(_step, can_not_delete):
    to_delete_css = 'a.remove-user'
    if can_not_delete:
        assert world.is_css_not_present(to_delete_css)
    else:
        assert world.is_css_present(to_delete_css)


@step(u'I can(not)? add users$')
@step(u's?he can(not)? add users$')
def can_add_users(_step, can_not_add):
    add_css = 'a.create-user-button'
    if can_not_add:
        assert world.is_css_not_present(add_css)
    else:
        assert world.is_css_present(add_css)


@step(u'I can(not)? make ("([^"]*)"|myself) a course team admin$')
@step(u's?he can(not)? make ("([^"]*)"|me) a course team admin$')
def can_make_course_admin(_step, can_not_make_admin, outer_capture, name):
    if outer_capture == "myself":
        email = world.scenario_dict["USER"].email
    else:
        email = name + '@edx.org'
    add_button_css = '.user-item[data-email="{email}"] .add-admin-role'.format(email=email)
    if can_not_make_admin:
        assert world.is_css_not_present(add_button_css)
    else:
        assert world.is_css_present(add_button_css)

########NEW FILE########
__FILENAME__ = course-updates
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from selenium.webdriver.common.keys import Keys
from common import type_in_codemirror, get_codemirror_value
from nose.tools import assert_in  # pylint: disable=E0611


@step(u'I go to the course updates page')
def go_to_updates(_step):
    menu_css = 'li.nav-course-courseware'
    updates_css = 'li.nav-course-courseware-updates a'
    world.css_click(menu_css)
    world.css_click(updates_css)


@step(u'I add a new update with the text "([^"]*)"$')
def add_update(_step, text):
    update_css = 'a.new-update-button'
    world.css_click(update_css)
    change_text(text)


@step(u'I should see the update "([^"]*)"$')
def check_update(_step, text):
    update_css = 'div.update-contents'
    update_html = world.css_find(update_css).html
    assert_in(text, update_html)


@step(u'I should not see the update "([^"]*)"$')
def check_no_update(_step, text):
    update_css = 'div.update-contents'
    assert world.is_css_not_present(update_css)


@step(u'I modify the text to "([^"]*)"$')
def modify_update(_step, text):
    button_css = 'div.post-preview a.edit-button'
    world.css_click(button_css)
    change_text(text)


@step(u'I change the update from "([^"]*)" to "([^"]*)"$')
def change_existing_update(_step, before, after):
    verify_text_in_editor_and_update('div.post-preview a.edit-button', before, after)


@step(u'I change the handout from "([^"]*)" to "([^"]*)"$')
def change_existing_handout(_step, before, after):
    verify_text_in_editor_and_update('div.course-handouts a.edit-button', before, after)


@step(u'I delete the update$')
def click_button(_step):
    button_css = 'div.post-preview a.delete-button'
    world.css_click(button_css)


@step(u'I edit the date to "([^"]*)"$')
def change_date(_step, new_date):
    button_css = 'div.post-preview a.edit-button'
    world.css_click(button_css)
    date_css = 'input.date'
    date = world.css_find(date_css)
    for i in range(len(date.value)):
        date._element.send_keys(Keys.END, Keys.BACK_SPACE)
    date._element.send_keys(new_date)
    save_css = 'a.save-button'
    world.css_click(save_css)


@step(u'I should see the date "([^"]*)"$')
def check_date(_step, date):
    date_css = 'span.date-display'
    assert_in(date, world.css_html(date_css))


@step(u'I modify the handout to "([^"]*)"$')
def edit_handouts(_step, text):
    edit_css = 'div.course-handouts > a.edit-button'
    world.css_click(edit_css)
    change_text(text)


@step(u'I see the handout "([^"]*)"$')
def check_handout(_step, handout):
    handout_css = 'div.handouts-content'
    assert_in(handout, world.css_html(handout_css))


@step(u'I see the handout error text')
def check_handout_error(_step):
    handout_error_css = 'div#handout_error'
    assert world.css_has_class(handout_error_css, 'is-shown')


@step(u'I see handout save button disabled')
def check_handout_error(_step):
    handout_save_button = 'form.edit-handouts-form a.save-button'
    assert world.css_has_class(handout_save_button, 'is-disabled')


@step(u'I edit the handout to "([^"]*)"$')
def edit_handouts(_step, text):
    type_in_codemirror(0, text)


@step(u'I see handout save button re-enabled')
def check_handout_error(_step):
    handout_save_button = 'form.edit-handouts-form a.save-button'
    assert not world.css_has_class(handout_save_button, 'is-disabled')


@step(u'I save handout edit')
def check_handout_error(_step):
    save_css = 'a.save-button'
    world.css_click(save_css)


def change_text(text):
    type_in_codemirror(0, text)
    save_css = 'a.save-button'
    world.css_click(save_css)


def verify_text_in_editor_and_update(button_css, before, after):
    world.css_click(button_css)
    text = get_codemirror_value()
    assert_in(before, text)
    change_text(after)

########NEW FILE########
__FILENAME__ = courses
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *

############### ACTIONS ####################


@step('There are no courses$')
def no_courses(step):
    world.clear_courses()
    create_studio_user()


@step('I click the New Course button$')
def i_click_new_course(step):
    world.css_click('.new-course-button')


@step('I fill in the new course information$')
def i_fill_in_a_new_course_information(step):
    fill_in_course_info()


@step('I create a course with "([^"]*)", "([^"]*)", "([^"]*)", and "([^"]*)"')
def i_create_course(step, name, org, number, run):
    fill_in_course_info(name=name, org=org, num=number, run=run)


@step('I create a new course$')
def i_create_a_course(step):
    create_a_course()


@step('I click the course link in My Courses$')
def i_click_the_course_link_in_my_courses(step):
    course_css = 'a.course-link'
    world.css_click(course_css)


@step('I see an error about the length of the org/course/run tuple')
def i_see_error_about_length(step):
    assert world.css_has_text('#course_creation_error', 'The combined length of the organization, course number, and course run fields cannot be more than 65 characters.')

############ ASSERTIONS ###################


@step('the Courseware page has loaded in Studio$')
def courseware_page_has_loaded_in_studio(step):
    course_title_css = 'span.course-title'
    assert world.is_css_present(course_title_css)


@step('I see the course listed in My Courses$')
def i_see_the_course_in_my_courses(step):
    course_css = 'h3.class-title'
    assert world.css_has_text(course_css, world.scenario_dict['COURSE'].display_name)


@step('I am on the "([^"]*)" tab$')
def i_am_on_tab(step, tab_name):
    header_css = 'div.inner-wrapper h1'
    assert world.css_has_text(header_css, tab_name)


@step('I see a link for adding a new section$')
def i_see_new_section_link(step):
    link_css = 'a.new-courseware-section-button'
    assert world.css_has_text(link_css, 'New Section')

########NEW FILE########
__FILENAME__ = course_import
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

import os
from lettuce import world, step
from django.conf import settings


def import_file(filename):
    world.browser.execute_script("$('input.file-input').css('display', 'block')")
    path = os.path.join(settings.COMMON_TEST_DATA_ROOT, "imports", filename)
    world.browser.attach_file('course-data', os.path.abspath(path))
    world.css_click('input.submit-button')
    # Go to course outline
    world.click_course_content()
    outline_css = 'li.nav-course-courseware-outline a'
    world.css_click(outline_css)


@step('I go to the import page$')
def go_to_import(step):
    menu_css = 'li.nav-course-tools'
    import_css = 'li.nav-course-tools-import a'
    world.css_click(menu_css)
    world.css_click(import_css)

########NEW FILE########
__FILENAME__ = discussion-editor
# disable missing docstring
# pylint: disable=C0111

from lettuce import world, step


@step('I have created a Discussion Tag$')
def i_created_discussion_tag(step):
    world.create_course_with_unit()
    world.create_component_instance(
        step=step,
        category='discussion',
    )


@step('I see three alphabetized settings and their expected values$')
def i_see_only_the_settings_and_values(step):
    world.verify_all_setting_entries(
        [
            ['Category', "Week 1", False],
            ['Display Name', "Discussion", False],
            ['Subcategory', "Topic-Level Student-Visible Label", False]
        ])


@step('I edit the component$')
def i_edit_and_select_settings(_step):
    world.edit_component()

########NEW FILE########
__FILENAME__ = grading
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from terrain.steps import reload_the_page
from selenium.common.exceptions import InvalidElementStateException
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_course_url
from nose.tools import assert_in, assert_not_in, assert_equal, assert_not_equal  # pylint: disable=E0611


@step(u'I am viewing the grading settings')
def view_grading_settings(step):
    world.click_course_settings()
    link_css = 'li.nav-course-settings-grading a'
    world.css_click(link_css)


@step(u'I add "([^"]*)" new grade')
def add_grade(step, many):
    grade_css = '.new-grade-button'
    for i in range(int(many)):
        world.css_click(grade_css)


@step(u'I delete a grade')
def delete_grade(step):
    #grade_css = 'li.grade-specific-bar > a.remove-button'
    #range_css = '.grade-specific-bar'
    #world.css_find(range_css)[1].mouseover()
    #world.css_click(grade_css)
    world.browser.execute_script('document.getElementsByClassName("remove-button")[0].click()')


@step(u'I see I now have "([^"]*)" grades$')
def view_grade_slider(step, how_many):
    grade_slider_css = '.grade-specific-bar'
    all_grades = world.css_find(grade_slider_css)
    assert_equal(len(all_grades), int(how_many))


@step(u'I move a grading section')
def move_grade_slider(step):
    moveable_css = '.ui-resizable-e'
    f = world.css_find(moveable_css).first
    f.action_chains.drag_and_drop_by_offset(f._element, 100, 0).perform()


@step(u'I see that the grade range has changed')
def confirm_change(step):
    range_css = '.range'
    all_ranges = world.css_find(range_css)
    for i in range(len(all_ranges)):
        assert_not_equal(world.css_html(range_css, index=i), '0-50')


@step(u'I change assignment type "([^"]*)" to "([^"]*)"$')
def change_assignment_name(step, old_name, new_name):
    name_id = '#course-grading-assignment-name'
    index = get_type_index(old_name)
    f = world.css_find(name_id)[index]
    assert_not_equal(index, -1)
    for count in range(len(old_name)):
        f._element.send_keys(Keys.END, Keys.BACK_SPACE)
    f._element.send_keys(new_name)


@step(u'I go back to the main course page')
def main_course_page(step):
    course_name = world.scenario_dict['COURSE'].display_name.replace(' ', '_')
    course_key = SlashSeparatedCourseKey(
        world.scenario_dict['COURSE'].org,
        world.scenario_dict['COURSE'].number,
        course_name
    )
    main_page_link = reverse_course_url('course_handler', course_key)

    world.visit(main_page_link)
    assert_in('Course Outline', world.css_text('h1.page-header'))


@step(u'I do( not)? see the assignment name "([^"]*)"$')
def see_assignment_name(step, do_not, name):
    assignment_menu_css = 'ul.menu > li > a'
    # First assert that it is there, make take a bit to redraw
    assert_true(
        world.css_find(assignment_menu_css),
        msg="Could not find assignment menu"
    )

    assignment_menu = world.css_find(assignment_menu_css)
    allnames = [item.html for item in assignment_menu]
    if do_not:
        assert_not_in(name, allnames)
    else:
        assert_in(name, allnames)


@step(u'I delete the assignment type "([^"]*)"$')
def delete_assignment_type(step, to_delete):
    delete_css = '.remove-grading-data'
    world.css_click(delete_css, index=get_type_index(to_delete))


@step(u'I add a new assignment type "([^"]*)"$')
def add_assignment_type(step, new_name):
    add_button_css = '.add-grading-data'
    world.css_click(add_button_css)
    name_id = '#course-grading-assignment-name'
    new_assignment = world.css_find(name_id)[-1]
    new_assignment._element.send_keys(new_name)


@step(u'I set the assignment weight to "([^"]*)"$')
def set_weight(step, weight):
    weight_id = '#course-grading-assignment-gradeweight'
    weight_field = world.css_find(weight_id)[-1]
    old_weight = world.css_value(weight_id, -1)
    for count in range(len(old_weight)):
        weight_field._element.send_keys(Keys.END, Keys.BACK_SPACE)
    weight_field._element.send_keys(weight)


@step(u'the assignment weight is displayed as "([^"]*)"$')
def verify_weight(step, weight):
    weight_id = '#course-grading-assignment-gradeweight'
    assert_equal(world.css_value(weight_id, -1), weight)


@step(u'I have populated the course')
def populate_course(step):
    step.given('I have added a new section')
    step.given('I have added a new subsection')


@step(u'I do not see the changes persisted on refresh$')
def changes_not_persisted(step):
    reload_the_page(step)
    name_id = '#course-grading-assignment-name'
    assert_equal(world.css_value(name_id), 'Homework')


@step(u'I see the assignment type "(.*)"$')
def i_see_the_assignment_type(_step, name):
    assignment_css = '#course-grading-assignment-name'
    assignments = world.css_find(assignment_css)
    types = [ele['value'] for ele in assignments]
    assert_in(name, types)


@step(u'I change the highest grade range to "(.*)"$')
def change_grade_range(_step, range_name):
    range_css = 'span.letter-grade'
    grade = world.css_find(range_css).first
    grade.value = range_name


@step(u'I see the highest grade range is "(.*)"$')
def i_see_highest_grade_range(_step, range_name):
    range_css = 'span.letter-grade'
    grade = world.css_find(range_css).first
    assert_equal(grade.value, range_name)


@step(u'I cannot edit the "Fail" grade range$')
def cannot_edit_fail(_step):
    range_css = 'span.letter-grade'
    ranges = world.css_find(range_css)
    assert_equal(len(ranges), 2)
    assert_not_equal(ranges.last.value, 'Failure')

    # try to change the grade range -- this should throw an exception
    try:
        ranges.last.value = 'Failure'
    except (InvalidElementStateException):
        pass  # We should get this exception on failing to edit the element

    # check to be sure that nothing has changed
    ranges = world.css_find(range_css)
    assert_equal(len(ranges), 2)
    assert_not_equal(ranges.last.value, 'Failure')


@step(u'I change the grace period to "(.*)"$')
def i_change_grace_period(_step, grace_period):
    grace_period_css = '#course-grading-graceperiod'
    ele = world.css_find(grace_period_css).first

    # Sometimes it takes a moment for the JavaScript
    # to populate the field.  If we don't wait for
    # this to happen, then we can end up with
    # an invalid value (e.g. "00:0048:00")
    # which prevents us from saving.
    assert_true(world.css_has_value(grace_period_css, "00:00"))

    # Set the new grace period
    ele.value = grace_period


@step(u'I see the grace period is "(.*)"$')
def the_grace_period_is(_step, grace_period):
    grace_period_css = '#course-grading-graceperiod'

    # The default value is 00:00
    # so we need to wait for it to change
    world.wait_for(
        lambda _: world.css_has_value(grace_period_css, grace_period)
    )


def get_type_index(name):
    name_id = '#course-grading-assignment-name'
    all_types = world.css_find(name_id)
    for index in range(len(all_types)):
        if world.css_value(name_id, index=index) == name:
            return index
    return -1

########NEW FILE########
__FILENAME__ = help
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

from nose.tools import assert_false  # pylint: disable=no-name-in-module
from lettuce import step, world


@step(u'I should see online help for "([^"]*)"$')
def see_online_help_for(step, page_name):
    # make sure the online Help link exists on this page and contains the expected page name
    elements_found = world.browser.find_by_xpath(
        '//li[contains(@class, "nav-account-help")]//a[contains(@href, "{page_name}")]'.format(
            page_name=page_name
        )
    )
    assert_false(elements_found.is_empty())

    # make sure the PDF link on the sock of this page exists
    # for now, the PDF link stays constant for all the pages so we just check for "pdf"
    elements_found = world.browser.find_by_xpath(
        '//section[contains(@class, "sock")]//li[contains(@class, "js-help-pdf")]//a[contains(@href, "pdf")]'
    )
    assert_false(elements_found.is_empty())

########NEW FILE########
__FILENAME__ = html-editor
# disable missing docstring
# pylint: disable=C0111

from lettuce import world, step
from nose.tools import assert_in, assert_false, assert_true, assert_equal  # pylint: disable=no-name-in-module
from common import type_in_codemirror, get_codemirror_value

CODEMIRROR_SELECTOR_PREFIX = "$('iframe').contents().find"


@step('I have created a Blank HTML Page$')
def i_created_blank_html_page(step):
    world.create_course_with_unit()
    world.create_component_instance(
        step=step,
        category='html',
        component_type='Text'
    )


@step('I have created a raw HTML component')
def i_created_raw_html(step):
    world.create_course_with_unit()
    world.create_component_instance(
        step=step,
        category='html',
        component_type='Raw HTML'
    )


@step('I see the HTML component settings$')
def i_see_only_the_html_display_name(step):
    world.verify_all_setting_entries(
        [
            ['Display Name', "Text", False],
            ['Editor', "Visual", False]
        ]
    )


@step('I have created an E-text Written in LaTeX$')
def i_created_etext_in_latex(step):
    world.create_course_with_unit()
    step.given('I have enabled latex compiler')
    world.create_component_instance(
        step=step,
        category='html',
        component_type='E-text Written in LaTeX'
    )


@step('I edit the page$')
def i_click_on_edit_icon(step):
    world.edit_component()


@step('I add an image with static link "(.*)" via the Image Plugin Icon$')
def i_click_on_image_plugin_icon(step, path):
    use_plugin(
        '.mce-i-image',
        lambda: world.css_fill('.mce-textbox', path, 0)
    )


@step('the link is shown as "(.*)" in the Image Plugin$')
def check_link_in_image_plugin(step, path):
    use_plugin(
        '.mce-i-image',
        lambda: assert_equal(path, world.css_find('.mce-textbox')[0].value)
    )


@step('I add a link with static link "(.*)" via the Link Plugin Icon$')
def i_click_on_link_plugin_icon(step, path):
    def fill_in_link_fields():
        world.css_fill('.mce-textbox', path, 0)
        world.css_fill('.mce-textbox', 'picture', 1)

    use_plugin('.mce-i-link', fill_in_link_fields)


@step('the link is shown as "(.*)" in the Link Plugin$')
def check_link_in_link_plugin(step, path):
    # Ensure caret position is within the link just created.
    script = """
    var editor = tinyMCE.activeEditor;
    editor.selection.select(editor.dom.select('a')[0]);"""
    world.browser.driver.execute_script(script)
    world.wait_for_ajax_complete()

    use_plugin(
        '.mce-i-link',
        lambda: assert_equal(path, world.css_find('.mce-textbox')[0].value)
    )


@step('type "(.*)" in the code editor and press OK$')
def type_in_codemirror_plugin(step, text):
    # Verify that raw code editor is not visible.
    assert_true(world.css_has_class('.CodeMirror', 'is-inactive'))
    # Verify that TinyMCE editor is present
    assert_true(world.is_css_present('.tiny-mce'))
    use_code_editor(
        lambda: type_in_codemirror(0, text, CODEMIRROR_SELECTOR_PREFIX)
    )


@step('and the code editor displays "(.*)"$')
def verify_code_editor_text(step, text):
    use_code_editor(
        lambda: assert_equal(text, get_codemirror_value(0, CODEMIRROR_SELECTOR_PREFIX))
    )


def use_plugin(button_class, action):
    # Click on plugin button
    world.css_click(button_class)
    perform_action_in_plugin(action)


def use_code_editor(action):
    # Click on plugin button
    buttons = world.css_find('div.mce-widget>button')

    code_editor = [button for button in buttons if button.text == 'HTML']
    assert_equal(1, len(code_editor))
    code_editor[0].click()

    perform_action_in_plugin(action)


def perform_action_in_plugin(action):
     # Wait for the plugin window to open.
    world.wait_for_visible('.mce-window')

    # Trigger the action
    action()

    # Click OK
    world.css_click('.mce-primary')


@step('I save the page$')
def i_click_on_save(step):
    world.save_component()


@step('the page text contains:')
def check_page_text(step):
    assert_in(step.multiline, world.css_find('.xmodule_HtmlModule').html)


@step('the Raw Editor contains exactly:')
def check_raw_editor_text(step):
    assert_equal(step.multiline, get_codemirror_value(0))


@step('the src link is rewritten to "(.*)"$')
def image_static_link_is_rewritten(step, path):
    # Find the TinyMCE iframe within the main window
    with world.browser.get_iframe('mce_0_ifr') as tinymce:
        image = tinymce.find_by_tag('img').first
        assert_in(path, image['src'])


@step('the href link is rewritten to "(.*)"$')
def link_static_link_is_rewritten(step, path):
    # Find the TinyMCE iframe within the main window
    with world.browser.get_iframe('mce_0_ifr') as tinymce:
        link = tinymce.find_by_tag('a').first
        assert_in(path, link['href'])


@step('the expected toolbar buttons are displayed$')
def check_toolbar_buttons(step):
    dropdowns = world.css_find('.mce-listbox')
    assert_equal(2, len(dropdowns))

    # Format dropdown
    assert_equal('Paragraph', dropdowns[0].text)
    # Font dropdown
    assert_equal('Font Family', dropdowns[1].text)

    buttons = world.css_find('.mce-ico')

    # Note that the code editor icon is not present because we are now showing text instead of an icon.
    # However, other test points user the code editor, so we have already verified its presence.
    expected_buttons = [
        'bold',
        'italic',
        'underline',
        'forecolor',
        # This is our custom "code style" button, which uses an image instead of a class.
        'none',
        'bullist',
        'numlist',
        'outdent',
        'indent',
        'blockquote',
        'link',
        'unlink',
        'image'
    ]

    assert_equal(len(expected_buttons), len(buttons))

    for index, button in enumerate(expected_buttons):
        class_names = buttons[index]._element.get_attribute('class')
        assert_equal("mce-ico mce-i-" + button, class_names)


@step('I set the text to "(.*)" and I select the text$')
def set_text_and_select(step, text):
    script = """
    var editor = tinyMCE.activeEditor;
    editor.setContent(arguments[0]);
    editor.selection.select(editor.dom.select('p')[0]);"""
    world.browser.driver.execute_script(script, str(text))
    world.wait_for_ajax_complete()


@step('I select the code toolbar button$')
def select_code_button(step):
    # This is our custom "code style" button. It uses an image instead of a class.
    world.css_click(".mce-i-none")


@step('type "(.*)" into the Raw Editor$')
def type_in_raw_editor(step, text):
    # Verify that CodeMirror editor is not hidden
    assert_false(world.css_has_class('.CodeMirror', 'is-inactive'))
    # Verify that TinyMCE Editor is not present
    assert_true(world.is_css_not_present('.tiny-mce'))
    type_in_codemirror(0, text)


@step('I edit the component and select the (Raw|Visual) Editor$')
def select_editor(step, editor):
    world.edit_component_and_select_settings()
    world.browser.select('Editor', editor)

########NEW FILE########
__FILENAME__ = pages
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

from lettuce import world, step
from nose.tools import assert_equal, assert_in  # pylint: disable=E0611


CSS_FOR_TAB_ELEMENT = "li[data-tab-id='{0}'] input.toggle-checkbox"


@step(u'I go to the pages page$')
def go_to_static(step):
    menu_css = 'li.nav-course-courseware'
    static_css = 'li.nav-course-courseware-pages a'
    world.css_click(menu_css)
    world.css_click(static_css)


@step(u'I add a new static page$')
def add_page(step):
    button_css = 'a.new-button'
    world.css_click(button_css)


@step(u'I should see a static page named "([^"]*)"$')
def see_a_static_page_named_foo(step, name):
    pages_css = 'div.xmodule_StaticTabModule'
    page_name_html = world.css_html(pages_css)
    assert_equal(page_name_html, '\n    {name}\n'.format(name=name))


@step(u'I should not see any static pages$')
def not_see_any_static_pages(step):
    pages_css = 'div.xmodule_StaticTabModule'
    assert (world.is_css_not_present(pages_css, wait_time=30))


@step(u'I "(edit|delete)" the static page$')
def click_edit_or_delete(step, edit_or_delete):
    button_css = 'ul.component-actions a.%s-button' % edit_or_delete
    world.css_click(button_css)


@step(u'I change the name to "([^"]*)"$')
def change_name(step, new_name):
    settings_css = '.settings-button'
    world.css_click(settings_css)
    input_css = 'input.setting-input'
    world.css_fill(input_css, new_name)
    if world.is_firefox():
        world.trigger_event(input_css)
    world.save_component()


@step(u'I drag the first static page to the last$')
def drag_first_static_page_to_last(step):
    drag_first_to_last_with_css('.component')


@step(u'I have created a static page$')
def create_static_page(step):
    step.given('I have opened the pages page in a new course')
    step.given('I add a new static page')


@step(u'I have opened the pages page in a new course$')
def open_pages_page_in_new_course(step):
    step.given('I have opened a new course in Studio')
    step.given('I go to the pages page')


@step(u'I have created two different static pages$')
def create_two_pages(step):
    step.given('I have created a static page')
    step.given('I "edit" the static page')
    step.given('I change the name to "First"')
    step.given('I add a new static page')
    # Verify order of pages
    _verify_page_names('First', 'Empty')


@step(u'the static pages are switched$')
def static_pages_are_switched(step):
    _verify_page_names('Empty', 'First')


def _verify_page_names(first, second):
    world.wait_for(
        func=lambda _: len(world.css_find('.xmodule_StaticTabModule')) == 2,
        timeout=200,
        timeout_msg="Timed out waiting for two pages to be present"
    )
    pages = world.css_find('.xmodule_StaticTabModule')
    assert_equal(pages[0].text, first)
    assert_equal(pages[1].text, second)


@step(u'the built-in pages are in the default order$')
def built_in_pages_in_default_order(step):
    expected_pages = ['Courseware', 'Course Info', 'Wiki', 'Progress']
    see_pages_in_expected_order(expected_pages)


@step(u'the built-in pages are switched$')
def built_in_pages_switched(step):
    expected_pages = ['Courseware', 'Course Info', 'Progress', 'Wiki']
    see_pages_in_expected_order(expected_pages)


@step(u'the pages are in the default order$')
def pages_in_default_order(step):
    expected_pages = ['Courseware', 'Course Info', 'Wiki', 'Progress', 'First', 'Empty']
    see_pages_in_expected_order(expected_pages)


@step(u'the pages are switched$$')
def pages_are_switched(step):
    expected_pages = ['Courseware', 'Course Info', 'Progress', 'First', 'Empty', 'Wiki']
    see_pages_in_expected_order(expected_pages)


@step(u'I drag the first page to the last$')
def drag_first_page_to_last(step):
    drag_first_to_last_with_css('.is-movable')


@step(u'I should see the "([^"]*)" page as "(visible|hidden)"$')
def page_is_visible_or_hidden(step, page_id, visible_or_hidden):
    hidden = visible_or_hidden == "hidden"
    assert_equal(world.css_find(CSS_FOR_TAB_ELEMENT.format(page_id)).checked, hidden)


@step(u'I toggle the visibility of the "([^"]*)" page$')
def page_toggle_visibility(step, page_id):
    world.css_find(CSS_FOR_TAB_ELEMENT.format(page_id))[0].click()


def drag_first_to_last_with_css(css_class):
    # For some reason, the drag_and_drop method did not work in this case.
    draggables = world.css_find(css_class + ' .drag-handle')
    source = draggables.first
    target = draggables.last
    source.action_chains.click_and_hold(source._element).perform()  # pylint: disable=protected-access
    source.action_chains.move_to_element_with_offset(target._element, 0, 50).perform()  # pylint: disable=protected-access
    source.action_chains.release().perform()


def see_pages_in_expected_order(page_names_in_expected_order):
    pages = world.css_find("li.course-tab")
    assert_equal(len(page_names_in_expected_order), len(pages))
    for i, page_name in enumerate(page_names_in_expected_order):
        assert_in(page_name, pages[i].text)

########NEW FILE########
__FILENAME__ = problem-editor
# disable missing docstring
# pylint: disable=C0111

import json
from lettuce import world, step
from nose.tools import assert_equal, assert_true  # pylint: disable=E0611
from common import type_in_codemirror, open_new_course
from advanced_settings import change_value
from course_import import import_file

DISPLAY_NAME = "Display Name"
MAXIMUM_ATTEMPTS = "Maximum Attempts"
PROBLEM_WEIGHT = "Problem Weight"
RANDOMIZATION = 'Randomization'
SHOW_ANSWER = "Show Answer"
TIMER_BETWEEN_ATTEMPTS = "Timer Between Attempts"

@step('I have created a Blank Common Problem$')
def i_created_blank_common_problem(step):
    world.create_course_with_unit()
    step.given("I have created another Blank Common Problem")


@step('I have created another Blank Common Problem$')
def i_create_new_common_problem(step):
    world.create_component_instance(
        step=step,
        category='problem',
        component_type='Blank Common Problem'
    )


@step('I edit and select Settings$')
def i_edit_and_select_settings(_step):
    world.edit_component_and_select_settings()


@step('I see the advanced settings and their expected values$')
def i_see_advanced_settings_with_values(step):
    world.verify_all_setting_entries(
        [
            [DISPLAY_NAME, "Blank Common Problem", True],
            [MAXIMUM_ATTEMPTS, "", False],
            [PROBLEM_WEIGHT, "", False],
            [RANDOMIZATION, "Never", False],
            [SHOW_ANSWER, "Finished", False],
            [TIMER_BETWEEN_ATTEMPTS, "0", False]
        ])


@step('I can modify the display name')
def i_can_modify_the_display_name(_step):
    # Verifying that the display name can be a string containing a floating point value
    # (to confirm that we don't throw an error because it is of the wrong type).
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, '3.4')
    verify_modified_display_name()


@step('my display name change is persisted on save')
def my_display_name_change_is_persisted_on_save(step):
    world.save_component_and_reopen(step)
    verify_modified_display_name()


@step('the problem display name is "(.*)"$')
def verify_problem_display_name(step, name):
    assert_equal(name.upper(), world.browser.find_by_css('.problem-header').text)


@step('I can specify special characters in the display name')
def i_can_modify_the_display_name_with_special_chars(_step):
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, "updated ' \" &")
    verify_modified_display_name_with_special_chars()


@step('my special characters and persisted on save')
def special_chars_persisted_on_save(step):
    world.save_component_and_reopen(step)
    verify_modified_display_name_with_special_chars()


@step('I can revert the display name to unset')
def can_revert_display_name_to_unset(_step):
    world.revert_setting_entry(DISPLAY_NAME)
    verify_unset_display_name()


@step('my display name is unset on save')
def my_display_name_is_persisted_on_save(step):
    world.save_component_and_reopen(step)
    verify_unset_display_name()


@step('I can select Per Student for Randomization')
def i_can_select_per_student_for_randomization(_step):
    world.browser.select(RANDOMIZATION, "Per Student")
    verify_modified_randomization()


@step('my change to randomization is persisted')
def my_change_to_randomization_is_persisted(step):
    world.save_component_and_reopen(step)
    verify_modified_randomization()


@step('I can revert to the default value for randomization')
def i_can_revert_to_default_for_randomization(step):
    world.revert_setting_entry(RANDOMIZATION)
    world.save_component_and_reopen(step)
    world.verify_setting_entry(world.get_setting_entry(RANDOMIZATION), RANDOMIZATION, "Never", False)


@step('I can set the weight to "(.*)"?')
def i_can_set_weight(_step, weight):
    set_weight(weight)
    verify_modified_weight()


@step('my change to weight is persisted')
def my_change_to_weight_is_persisted(step):
    world.save_component_and_reopen(step)
    verify_modified_weight()


@step('I can revert to the default value of unset for weight')
def i_can_revert_to_default_for_unset_weight(step):
    world.revert_setting_entry(PROBLEM_WEIGHT)
    world.save_component_and_reopen(step)
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "", False)


@step('if I set the weight to "(.*)", it remains unset')
def set_the_weight_to_abc(step, bad_weight):
    set_weight(bad_weight)
    # We show the clear button immediately on type, hence the "True" here.
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "", True)
    world.save_component_and_reopen(step)
    # But no change was actually ever sent to the model, so on reopen, explicitly_set is False
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "", False)


@step('if I set the max attempts to "(.*)", it will persist as a valid integer$')
def set_the_max_attempts(step, max_attempts_set):
    # on firefox with selenium, the behavior is different.
    # eg 2.34 displays as 2.34 and is persisted as 2
    index = world.get_setting_entry_index(MAXIMUM_ATTEMPTS)
    world.set_field_value(index, max_attempts_set)
    world.save_component_and_reopen(step)
    value = world.css_value('input.setting-input', index=index)
    assert value != "", "max attempts is blank"
    assert int(value) >= 0


@step('Edit High Level Source is not visible')
def edit_high_level_source_not_visible(step):
    verify_high_level_source_links(step, False)


@step('Edit High Level Source is visible')
def edit_high_level_source_links_visible(step):
    verify_high_level_source_links(step, True)


@step('If I press Cancel my changes are not persisted')
def cancel_does_not_save_changes(step):
    world.cancel_component(step)
    step.given("I edit and select Settings")
    step.given("I see the advanced settings and their expected values")


@step('I have enabled latex compiler')
def enable_latex_compiler(step):
    url = world.browser.url
    step.given("I select the Advanced Settings")
    change_value(step, 'use_latex_compiler', 'true')
    world.visit(url)
    world.wait_for_xmodule()


@step('I have created a LaTeX Problem')
def create_latex_problem(step):
    world.create_course_with_unit()
    step.given('I have enabled latex compiler')
    world.create_component_instance(
        step=step,
        category='problem',
        component_type='Problem Written in LaTeX',
        is_advanced=True
    )


@step('I edit and compile the High Level Source')
def edit_latex_source(_step):
    open_high_level_source()
    type_in_codemirror(1, "hi")
    world.css_click('.hls-compile')


@step('my change to the High Level Source is persisted')
def high_level_source_persisted(_step):
    def verify_text(driver):
        css_sel = '.problem div>span'
        return world.css_text(css_sel) == 'hi'

    world.wait_for(verify_text, timeout=10)


@step('I view the High Level Source I see my changes')
def high_level_source_in_editor(_step):
    open_high_level_source()
    assert_equal('hi', world.css_value('.source-edit-box'))


@step(u'I have an empty course')
def i_have_empty_course(step):
    open_new_course()


@step(u'I import the file "([^"]*)"$')
def i_import_the_file(_step, filename):
    import_file(filename)


@step(u'I go to the vertical "([^"]*)"$')
def i_go_to_vertical(_step, vertical):
    world.css_click("span:contains('{0}')".format(vertical))


@step(u'I go to the unit "([^"]*)"$')
def i_go_to_unit(_step, unit):
    loc = "window.location = $(\"span:contains('{0}')\").closest('a').attr('href')".format(unit)
    world.browser.execute_script(loc)


@step(u'I see a message that says "([^"]*)"$')
def i_can_see_message(_step, msg):
    msg = json.dumps(msg)     # escape quotes
    world.css_has_text("h2.title", msg)


@step(u'I can edit the problem$')
def i_can_edit_problem(_step):
    world.edit_component()


@step(u'I can see cheatsheet$')
def verify_cheat_sheet_displaying(_step):
    world.css_click("a.cheatsheet-toggle")
    css_selector = 'article.simple-editor-cheatsheet'
    world.wait_for_visible(css_selector)


def verify_high_level_source_links(step, visible):
    if visible:
        assert_true(world.is_css_present('.launch-latex-compiler'),
                    msg="Expected to find the latex button but it is not present.")
    else:
        assert_true(world.is_css_not_present('.launch-latex-compiler'),
                    msg="Expected not to find the latex button but it is present.")

    world.cancel_component(step)


def verify_modified_weight():
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "3.5", True)


def verify_modified_randomization():
    world.verify_setting_entry(world.get_setting_entry(RANDOMIZATION), RANDOMIZATION, "Per Student", True)


def verify_modified_display_name():
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, '3.4', True)


def verify_modified_display_name_with_special_chars():
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, "updated ' \" &", True)


def verify_unset_display_name():
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, 'Blank Advanced Problem', False)


def set_weight(weight):
    index = world.get_setting_entry_index(PROBLEM_WEIGHT)
    world.set_field_value(index, weight)


def open_high_level_source():
    world.edit_component()
    world.css_click('.launch-latex-compiler > a')

########NEW FILE########
__FILENAME__ = section
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from nose.tools import assert_equal  # pylint: disable=E0611


@step('I click the New Section link$')
def i_click_new_section_link(_step):
    link_css = 'a.new-courseware-section-button'
    world.css_click(link_css)


@step('I enter the section name and click save$')
def i_save_section_name(_step):
    save_section_name('My Section')


@step('I enter a section name with a quote and click save$')
def i_save_section_name_with_quote(_step):
    save_section_name('Section with "Quote"')


@step('I have added a new section$')
def i_have_added_new_section(_step):
    add_section()


@step('I click the Edit link for the release date$')
def i_click_the_edit_link_for_the_release_date(_step):
    button_css = 'div.section-published-date a.edit-release-date'
    world.css_click(button_css)


@step('I set the section release date to ([0-9/-]+)( [0-9:]+)?')
def set_section_release_date(_step, datestring, timestring):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    if not timestring:
        timestring = "00:00"
    set_date_and_time(
        'input.start-date.date.hasDatepicker', datestring,
        'input.start-time.time.ui-timepicker-input', timestring)
    world.browser.click_link_by_text('Save')


@step('I see a "(saving|deleting)" notification')
def i_see_a_mini_notification(_step, _type):
    saving_css = '.wrapper-notification-mini'
    assert world.is_css_present(saving_css)


@step('I see my section on the Courseware page$')
def i_see_my_section_on_the_courseware_page(_step):
    see_my_section_on_the_courseware_page('My Section')


@step('I see my section name with a quote on the Courseware page$')
def i_see_my_section_name_with_quote_on_the_courseware_page(_step):
    see_my_section_on_the_courseware_page('Section with "Quote"')


@step('I click to edit the section name$')
def i_click_to_edit_section_name(_step):
    world.css_click('span.section-name-span')


@step('I see the complete section name with a quote in the editor$')
def i_see_complete_section_name_with_quote_in_editor(_step):
    css = '.section-name-edit input[type=text]'
    assert world.is_css_present(css)
    assert_equal(world.css_value(css), 'Section with "Quote"')


@step('the section does not exist$')
def section_does_not_exist(_step):
    css = 'h3[data-name="My Section"]'
    assert world.is_css_not_present(css)


@step('I see a release date for my section$')
def i_see_a_release_date_for_my_section(_step):
    import re

    css = 'span.published-status'
    assert world.is_css_present(css)
    status_text = world.css_text(css)

    # e.g. 11/06/2012 at 16:25
    msg = 'Release date:'
    date_regex = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d\d?, \d{4}'
    if not re.search(date_regex, status_text):
        print status_text, date_regex
    time_regex = r'[0-2]\d:[0-5]\d( \w{3})?'
    if not re.search(time_regex, status_text):
        print status_text, time_regex
    match_string = r'%s\s+%s at %s' % (msg, date_regex, time_regex)
    if not re.match(match_string, status_text):
        print status_text, match_string
    assert re.match(match_string, status_text)


@step('I see a link to create a new subsection$')
def i_see_a_link_to_create_a_new_subsection(_step):
    css = 'a.new-subsection-item'
    assert world.is_css_present(css)


@step('the section release date picker is not visible$')
def the_section_release_date_picker_not_visible(_step):
    css = 'div.edit-subsection-publish-settings'
    assert not world.css_visible(css)


@step('the section release date is updated$')
def the_section_release_date_is_updated(_step):
    css = 'span.published-status'
    status_text = world.css_text(css)
    assert_equal(status_text, 'Release date: 12/25/2013 at 00:00 UTC')


def save_section_name(name):
    name_css = '.new-section-name'
    save_css = '.new-section-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)


def see_my_section_on_the_courseware_page(name):
    section_css = 'span.section-name-span'
    assert world.css_has_text(section_css, name)

########NEW FILE########
__FILENAME__ = signup
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step


@step('I fill in the registration form$')
def i_fill_in_the_registration_form(step):
    def fill_in_reg_form():
        register_form = world.css_find('form#register_form')
        register_form.find_by_name('email').fill('robot+studio@edx.org')
        register_form.find_by_name('password').fill('test')
        register_form.find_by_name('username').fill('robot-studio')
        register_form.find_by_name('name').fill('Robot Studio')
        register_form.find_by_name('terms_of_service').click()
    world.retry_on_exception(fill_in_reg_form)


@step('I press the Create My Account button on the registration form$')
def i_press_the_button_on_the_registration_form(step):
    submit_css = 'form#register_form button#submit'
    world.css_click(submit_css)


@step('I should see an email verification prompt')
def i_should_see_an_email_verification_prompt(step):
    world.css_has_text('h1.page-header', u'My Courses')
    world.css_has_text('div.msg h3.title', u'We need to verify your email address')


@step(u'I fill in and submit the signin form$')
def i_fill_in_the_signin_form(step):
    def fill_login_form():
        login_form = world.browser.find_by_css('form#login_form')
        login_form.find_by_name('email').fill('robot+studio@edx.org')
        login_form.find_by_name('password').fill('test')
        login_form.find_by_name('submit').click()
    world.retry_on_exception(fill_login_form)

########NEW FILE########
__FILENAME__ = subsection
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from nose.tools import assert_equal  # pylint: disable=E0611

############### ACTIONS ####################


@step('I have opened a new course section in Studio$')
def i_have_opened_a_new_course_section(step):
    open_new_course()
    add_section()


@step('I have added a new subsection$')
def i_have_added_a_new_subsection(step):
    add_subsection()


@step('I have opened a new subsection in Studio$')
def i_have_opened_a_new_subsection(step):
    step.given('I have opened a new course section in Studio')
    step.given('I have added a new subsection')
    world.css_click('span.subsection-name-value')


@step('I click the New Subsection link')
def i_click_the_new_subsection_link(step):
    world.css_click('a.new-subsection-item')


@step('I enter the subsection name and click save$')
def i_save_subsection_name(step):
    save_subsection_name('Subsection One')


@step('I enter a subsection name with a quote and click save$')
def i_save_subsection_name_with_quote(step):
    save_subsection_name('Subsection With "Quote"')


@step('I click on the subsection$')
def click_on_subsection(step):
    world.css_click('span.subsection-name-value')


@step('I see the complete subsection name with a quote in the editor$')
def i_see_complete_subsection_name_with_quote_in_editor(step):
    css = '.subsection-display-name-input'
    assert world.is_css_present(css)
    assert_equal(world.css_value(css), 'Subsection With "Quote"')


@step('I set the subsection release date to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_release_date(_step, datestring, timestring):
    set_subsection_date('input#start_date', datestring, 'input#start_time', timestring)


@step('I set the subsection release date on enter to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_release_date_on_enter(_step, datestring, timestring):  # pylint: disable-msg=invalid-name
    set_subsection_date('input#start_date', datestring, 'input#start_time', timestring, 'ENTER')


@step('I set the subsection due date to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_due_date(_step, datestring, timestring, key=None):
    if not world.css_visible('input#due_date'):
        world.css_click('.due-date-input .set-date')

    assert world.css_visible('input#due_date')
    set_subsection_date('input#due_date', datestring, 'input#due_time', timestring, key)


@step('I set the subsection due date on enter to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_due_date_on_enter(_step, datestring, timestring):  # pylint: disable-msg=invalid-name
    set_subsection_due_date(_step, datestring, timestring, 'ENTER')


@step('I mark it as Homework$')
def i_mark_it_as_homework(step):
    world.css_click('a.menu-toggle')
    world.browser.click_link_by_text('Homework')


@step('I see it marked as Homework$')
def i_see_it_marked__as_homework(step):
    assert_equal(world.css_value(".status-label"), 'Homework')


@step('I click the link to sync release date to section')
def click_sync_release_date(step):
    world.css_click('.sync-date')


############ ASSERTIONS ###################


@step('I see my subsection on the Courseware page$')
def i_see_my_subsection_on_the_courseware_page(step):
    see_subsection_name('Subsection One')


@step('I see my subsection name with a quote on the Courseware page$')
def i_see_my_subsection_name_with_quote_on_the_courseware_page(step):
    see_subsection_name('Subsection With "Quote"')


@step('the subsection does not exist$')
def the_subsection_does_not_exist(step):
    css = 'span.subsection-name'
    assert world.is_css_not_present(css)


@step('I see the subsection release date is ([0-9/-]+)( [0-9:]+)?')
def i_see_subsection_release(_step, datestring, timestring):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    assert_equal(datestring, get_date('input#start_date'))
    if timestring:
        assert_equal(timestring, get_date('input#start_time'))


@step('I see the subsection due date is ([0-9/-]+)( [0-9:]+)?')
def i_see_subsection_due(_step, datestring, timestring):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    assert_equal(datestring, get_date('input#due_date'))
    if timestring:
        assert_equal(timestring, get_date('input#due_time'))


############ HELPER METHODS ###################
def get_date(css):
    return world.css_find(css).first.value.strip()


def save_subsection_name(name):
    name_css = 'input.new-subsection-name-input'
    save_css = 'input.new-subsection-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)


def see_subsection_name(name):
    css = 'span.subsection-name'
    assert world.is_css_present(css)
    css = 'span.subsection-name-value'
    assert world.css_has_text(css, name)


def set_subsection_date(date_css, datestring, time_css, timestring, key=None):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    if not timestring:
        timestring = "00:00"

    set_date_and_time(date_css, datestring, time_css, timestring, key)

########NEW FILE########
__FILENAME__ = textbooks
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from django.conf import settings
from common import upload_file
from nose.tools import assert_equal

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT


@step(u'I go to the textbooks page')
def go_to_uploads(_step):
    world.wait_for_js_to_load()
    world.click_course_content()
    menu_css = 'li.nav-course-courseware-textbooks a'
    world.css_click(menu_css)


@step(u'I should see a message telling me to create a new textbook')
def assert_create_new_textbook_msg(_step):
    css = ".wrapper-content .no-textbook-content"
    assert world.is_css_present(css)
    no_tb = world.css_find(css)
    assert "You haven't added any textbooks" in no_tb.text


@step(u'I upload the textbook "([^"]*)"$')
def upload_textbook(_step, file_name):
    upload_file(file_name, sub_path="uploads/")


@step(u'I click (on )?the New Textbook button')
def click_new_textbook(_step, on):
    button_css = ".nav-actions .new-button"
    button = world.css_find(button_css)
    button.click()


@step(u'I name my textbook "([^"]*)"')
def name_textbook(_step, name):
    input_css = ".textbook input[name=textbook-name]"
    world.css_fill(input_css, name)
    if world.is_firefox():
        world.trigger_event(input_css)


@step(u'I name the (first|second|third) chapter "([^"]*)"')
def name_chapter(_step, ordinal, name):
    index = ["first", "second", "third"].index(ordinal)
    input_css = ".textbook .chapter{i} input.chapter-name".format(i=index + 1)
    world.css_fill(input_css, name)
    if world.is_firefox():
        world.trigger_event(input_css)


@step(u'I type in "([^"]*)" for the (first|second|third) chapter asset')
def asset_chapter(_step, name, ordinal):
    index = ["first", "second", "third"].index(ordinal)
    input_css = ".textbook .chapter{i} input.chapter-asset-path".format(i=index + 1)
    world.css_fill(input_css, name)
    if world.is_firefox():
        world.trigger_event(input_css)


@step(u'I click the Upload Asset link for the (first|second|third) chapter')
def click_upload_asset(_step, ordinal):
    index = ["first", "second", "third"].index(ordinal)
    button_css = ".textbook .chapter{i} .action-upload".format(i=index + 1)
    world.css_click(button_css)


@step(u'I click Add a Chapter')
def click_add_chapter(_step):
    button_css = ".textbook .action-add-chapter"
    world.css_click(button_css)


@step(u'I save the textbook')
def save_textbook(_step):
    submit_css = "form.edit-textbook button[type=submit]"
    world.css_click(submit_css)


@step(u'I should see a textbook named "([^"]*)" with a chapter path containing "([^"]*)"')
def check_textbook(_step, textbook_name, chapter_name):
    title = world.css_text(".textbook h3.textbook-title", index=0)
    chapter = world.css_text(".textbook .wrap-textbook p", index=0)
    assert_equal(title, textbook_name)
    assert_equal(chapter, chapter_name)


@step(u'I should see a textbook named "([^"]*)" with (\d+) chapters')
def check_textbook_chapters(_step, textbook_name, num_chapters_str):
    num_chapters = int(num_chapters_str)
    title = world.css_text(".textbook .view-textbook h3.textbook-title", index=0)
    toggle_text = world.css_text(".textbook .view-textbook .chapter-toggle", index=0)
    assert_equal(title, textbook_name)
    assert_equal(
        toggle_text,
        "{num} PDF Chapters".format(num=num_chapters),
        "Expected {num} chapters, found {real}".format(num=num_chapters, real=toggle_text)
    )


@step(u'I click the textbook chapters')
def click_chapters(_step):
    world.css_click(".textbook a.chapter-toggle")


@step(u'the (first|second|third) chapter should be named "([^"]*)"')
def check_chapter_name(_step, ordinal, name):
    index = ["first", "second", "third"].index(ordinal)
    chapter = world.css_find(".textbook .view-textbook ol.chapters li")[index]
    element = chapter.find_by_css(".chapter-name")
    assert element.text == name, "Expected chapter named {expected}, found chapter named {actual}".format(
        expected=name, actual=element.text)


@step(u'the (first|second|third) chapter should have an asset called "([^"]*)"')
def check_chapter_asset(_step, ordinal, name):
    index = ["first", "second", "third"].index(ordinal)
    chapter = world.css_find(".textbook .view-textbook ol.chapters li")[index]
    element = chapter.find_by_css(".chapter-asset-path")
    assert element.text == name, "Expected chapter with asset {expected}, found chapter with asset {actual}".format(
        expected=name, actual=element.text)

########NEW FILE########
__FILENAME__ = transcripts
# disable missing docstring
# pylint: disable=C0111

import os
from lettuce import world, step

from django.conf import settings

from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from splinter.request_handler.request_handler import RequestHandler

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT

# We should wait 300 ms for event handler invocation + 200ms for safety.
DELAY = 0.5

ERROR_MESSAGES = {
    'url_format': u'Incorrect url format.',
    'file_type': u'Link types should be unique.',
}

STATUSES = {
    'found': u'Timed Transcript Found',
    'not found on edx': u'No EdX Timed Transcript',
    'not found': u'No Timed Transcript',
    'replace': u'Timed Transcript Conflict',
    'uploaded_successfully': u'Timed Transcript Uploaded Successfully',
    'use existing': u'Confirm Timed Transcript',
}

SELECTORS = {
    'error_bar': '.transcripts-error-message',
    'url_inputs': '.videolist-settings-item input.input',
    'collapse_link': '.collapse-action.collapse-setting',
    'collapse_bar': '.videolist-extra-videos',
    'status_bar': '.transcripts-message-status',
}

# button type , button css selector, button message
TRANSCRIPTS_BUTTONS = {
    'import': ('.setting-import', 'Import YouTube Transcript'),
    'download_to_edit': ('.setting-download', 'Download Transcript for Editing'),
    'disabled_download_to_edit': ('.setting-download.is-disabled', 'Download Transcript for Editing'),
    'upload_new_timed_transcripts': ('.setting-upload',  'Upload New Transcript'),
    'replace': ('.setting-replace', 'Yes, replace the edX transcript with the YouTube transcript'),
    'choose': ('.setting-choose', 'Timed Transcript from {}'),
    'use_existing': ('.setting-use-existing', 'Use Current Transcript'),
}


@step('I clear fields$')
def clear_fields(_step):

    # Clear the input fields and trigger an 'input' event
    script = """
        $('{selector}')
            .prop('disabled', false)
            .removeClass('is-disabled')
            .val('')
            .trigger('input');
    """.format(selector=SELECTORS['url_inputs'])
    world.browser.execute_script(script)

    world.wait(DELAY)
    world.wait_for_ajax_complete()


@step('I clear field number (.+)$')
def clear_field(_step, index):
    index = int(index) - 1
    world.css_fill(SELECTORS['url_inputs'], '', index)

    # For some reason ChromeDriver doesn't trigger an 'input' event after filling
    # the field with an empty value. That's why we trigger it manually via jQuery.
    world.trigger_event(SELECTORS['url_inputs'], event='input', index=index)

    world.wait(DELAY)
    world.wait_for_ajax_complete()


@step('I expect (.+) inputs are disabled$')
def inputs_are_disabled(_step, indexes):
    index_list = [int(i.strip()) - 1 for i in indexes.split(',')]
    for index in index_list:
        el = world.css_find(SELECTORS['url_inputs'])[index]

        assert el['disabled']


@step('I expect inputs are enabled$')
def inputs_are_enabled(_step):
    for index in range(3):
        el = world.css_find(SELECTORS['url_inputs'])[index]

        assert not el['disabled']


@step('I do not see error message$')
def i_do_not_see_error_message(_step):
    assert not world.css_visible(SELECTORS['error_bar'])


@step('I see error message "([^"]*)"$')
def i_see_error_message(_step, error):
    assert world.css_has_text(SELECTORS['error_bar'], ERROR_MESSAGES[error])


@step('I do not see status message$')
def i_do_not_see_status_message(_step):
    assert not world.css_visible(SELECTORS['status_bar'])


@step('I see status message "([^"]*)"$')
def i_see_status_message(_step, status):
    assert not world.css_visible(SELECTORS['error_bar'])
    assert world.css_has_text(SELECTORS['status_bar'], STATUSES[status])

    DOWNLOAD_BUTTON = TRANSCRIPTS_BUTTONS["download_to_edit"][0]
    if world.is_css_present(DOWNLOAD_BUTTON, wait_time=1) \
    and not world.css_find(DOWNLOAD_BUTTON)[0].has_class('is-disabled'):
        assert _transcripts_are_downloaded()


@step('I (.*)see button "([^"]*)"$')
def i_see_button(_step, not_see, button_type):
    button = button_type.strip()

    if not_see.strip():
        assert world.is_css_not_present(TRANSCRIPTS_BUTTONS[button][0])
    else:
        assert world.css_has_text(TRANSCRIPTS_BUTTONS[button][0], TRANSCRIPTS_BUTTONS[button][1])


@step('I (.*)see (.*)button "([^"]*)" number (\d+)$')
def i_see_button_with_custom_text(_step, not_see, button_type, custom_text, index):
    button = button_type.strip()
    custom_text = custom_text.strip()
    index = int(index.strip()) - 1

    if not_see.strip():
        assert world.is_css_not_present(TRANSCRIPTS_BUTTONS[button][0])
    else:
        assert world.css_has_text(TRANSCRIPTS_BUTTONS[button][0], TRANSCRIPTS_BUTTONS[button][1].format(custom_text), index)


@step('I click transcript button "([^"]*)"$')
def click_button_transcripts_variant(_step, button_type):
    button = button_type.strip()
    world.css_click(TRANSCRIPTS_BUTTONS[button][0])
    world.wait_for_ajax_complete()


@step('I click transcript button "([^"]*)" number (\d+)$')
def click_button_index(_step, button_type, index):
    button = button_type.strip()
    index = int(index.strip()) - 1

    world.css_click(TRANSCRIPTS_BUTTONS[button][0], index)
    world.wait_for_ajax_complete()


@step('I remove "([^"]+)" transcripts id from store')
def remove_transcripts_from_store(_step, subs_id):
    """Remove from store, if transcripts content exists."""
    filename = 'subs_{0}.srt.sjson'.format(subs_id.strip())
    content_location = StaticContent.compute_location(
        world.scenario_dict['COURSE'].id,
        filename
    )
    try:
        content = contentstore().find(content_location)
        contentstore().delete(content.get_id())
        print('Transcript file was removed from store.')
    except NotFoundError:
        print('Transcript file was NOT found and not removed.')


@step('I enter a "([^"]+)" source to field number (\d+)$')
def i_enter_a_source(_step, link, index):
    index = int(index) - 1

    if index is not 0 and not world.css_visible(SELECTORS['collapse_bar']):
        world.css_click(SELECTORS['collapse_link'])

        assert world.css_visible(SELECTORS['collapse_bar'])

    world.css_fill(SELECTORS['url_inputs'], link, index)
    world.wait(DELAY)
    world.wait_for_ajax_complete()


@step('I upload the transcripts file "([^"]*)"$')
def upload_file(_step, file_name):
    path = os.path.join(TEST_ROOT, 'uploads/', file_name.strip())
    world.browser.execute_script("$('form.file-chooser').show()")
    world.browser.attach_file('transcript-file', os.path.abspath(path))
    world.wait_for_ajax_complete()


@step('I see "([^"]*)" text in the captions')
def check_text_in_the_captions(_step, text):
    world.wait_for_present('.video.is-captions-rendered')
    world.wait_for(lambda _: world.css_text('.subtitles'), timeout=30)
    actual_text = world.css_text('.subtitles')
    assert (text in actual_text)


@step('I see value "([^"]*)" in the field "([^"]*)"$')
def check_transcripts_field(_step, values, field_name):
    world.select_editor_tab('Advanced')
    tab = world.css_find('#settings-tab').first;
    field_id = '#' + tab.find_by_xpath('.//label[text()="%s"]' % field_name.strip())[0]['for']
    values_list = [i.strip() == world.css_value(field_id) for i in values.split('|')]
    assert any(values_list)
    world.select_editor_tab('Basic')


@step('I save changes$')
def save_changes(_step):
    world.save_component()


@step('I open tab "([^"]*)"$')
def open_tab(_step, tab_name):
    world.select_editor_tab(tab_name)


@step('I set value "([^"]*)" to the field "([^"]*)"$')
def set_value_transcripts_field(_step, value, field_name):
    tab = world.css_find('#settings-tab').first;
    XPATH = './/label[text()="{name}"]'.format(name=field_name)
    SELECTOR = '#' + tab.find_by_xpath(XPATH)[0]['for']
    element = world.css_find(SELECTOR).first
    if element['type'] == 'text':
        SCRIPT = '$("{selector}").val("{value}").change()'.format(
                selector=SELECTOR,
                value=value
            )
        world.browser.execute_script(SCRIPT)
        assert world.css_has_value(SELECTOR, value)
    else:
        assert False, 'Incorrect element type.';
    world.wait_for_ajax_complete()


@step('I revert the transcript field "([^"]*)"$')
def revert_transcripts_field(_step, field_name):
    world.revert_setting_entry(field_name)


def _transcripts_are_downloaded():
    world.wait_for_ajax_complete()
    request = RequestHandler()
    DOWNLOAD_BUTTON = world.css_find(TRANSCRIPTS_BUTTONS["download_to_edit"][0]).first
    url = DOWNLOAD_BUTTON['href']
    request.connect(url)

    return request.status_code.is_success()

########NEW FILE########
__FILENAME__ = upload
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from lettuce.django import django_url
from django.conf import settings
import requests
import string
import random
import os
from django.contrib.auth.models import User
from student.models import CourseEnrollment
from nose.tools import assert_equal, assert_not_equal  # pylint: disable=E0611

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT
ASSET_NAMES_CSS = 'td.name-col > span.title > a.filename'


@step(u'I go to the files and uploads page$')
def go_to_uploads(_step):
    menu_css = 'li.nav-course-courseware'
    uploads_css = 'li.nav-course-courseware-uploads a'
    world.css_click(menu_css)
    world.css_click(uploads_css)


@step(u'I upload the( test)? file "([^"]*)"$')
def upload_file(_step, is_test_file, file_name, button_text=None):
    if button_text:
        world.click_link(button_text)
    else:
        world.click_link('Upload New File')

    if not is_test_file:
        _write_test_file(file_name, "test file")

    # uploading the file itself
    path = os.path.join(TEST_ROOT, 'uploads/', file_name)
    world.browser.execute_script("$('input.file-input').css('display', 'block')")
    world.browser.attach_file('file', os.path.abspath(path))
    close_css = 'a.close-button'
    world.css_click(close_css)


@step(u'I upload the file "([^"]*)" by clicking "([^"]*)"')
def upload_file_on_button_press(_step, file_name, button_text=None):
    upload_file(_step, '', file_name, button_text)


@step(u'I upload the files "([^"]*)"$')
def upload_files(_step, files_string):
    # files_string should be comma separated with no spaces.
    files = files_string.split(",")
    upload_css = 'a.upload-button'
    world.css_click(upload_css)

    # uploading the files
    for filename in files:
        _write_test_file(filename, "test file")
        path = os.path.join(TEST_ROOT, 'uploads/', filename)
        world.browser.execute_script("$('input.file-input').css('display', 'block')")
        world.browser.attach_file('file', os.path.abspath(path))

    close_css = 'a.close-button'
    world.css_click(close_css)


@step(u'I should not see the file "([^"]*)" was uploaded$')
def check_not_there(_step, file_name):
    # Either there are no files, or there are files but
    # not the one I expect not to exist.

    # Since our only test for deletion right now deletes
    # the only file that was uploaded, our success criteria
    # will be that there are no files.
    # In the future we can refactor if necessary.
    assert(world.is_css_not_present(ASSET_NAMES_CSS))


@step(u'I should see the file "([^"]*)" was uploaded$')
def check_upload(_step, file_name):
    index = get_index(file_name)
    assert_not_equal(index, -1)


@step(u'The url for the file "([^"]*)" is valid$')
def check_url(_step, file_name):
    r = get_file(file_name)
    assert_equal(r.status_code, 200)


@step(u'I delete the file "([^"]*)"$')
def delete_file(_step, file_name):
    index = get_index(file_name)
    assert index != -1
    delete_css = "a.remove-asset-button"
    world.css_click(delete_css, index=index)
    world.confirm_studio_prompt()


@step(u'I should see only one "([^"]*)"$')
def no_duplicate(_step, file_name):
    all_names = world.css_find(ASSET_NAMES_CSS)
    only_one = False
    for i in range(len(all_names)):
        if file_name == world.css_html(ASSET_NAMES_CSS, index=i):
            only_one = not only_one
    assert only_one


@step(u'I can download the correct "([^"]*)" file$')
def check_download(_step, file_name):
    path = os.path.join(TEST_ROOT, 'uploads/', file_name)
    with open(os.path.abspath(path), 'r') as cur_file:
        cur_text = cur_file.read()
        r = get_file(file_name)
        downloaded_text = r.text
        assert cur_text == downloaded_text
    # resetting the file back to its original state
    _write_test_file(file_name, "This is an arbitrary file for testing uploads")


def _write_test_file(file_name, text):
    path = os.path.join(TEST_ROOT, 'uploads/', file_name)
    # resetting the file back to its original state
    with open(os.path.abspath(path), 'w') as cur_file:
        cur_file.write(text)


@step(u'I modify "([^"]*)"$')
def modify_upload(_step, file_name):
    new_text = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10))
    _write_test_file(file_name, new_text)


@step(u'I upload an asset$')
def upload_an_asset(step):
    step.given('I upload the file "asset.html"')


@step(u'I (lock|unlock) the asset$')
def lock_unlock_file(_step, _lock_state):
    index = get_index('asset.html')
    assert index != -1, 'Expected to find an asset but could not.'

    # Warning: this is a misnomer, it really only toggles the
    # lock state. TODO: fix it.
    lock_css = "input.lock-checkbox"
    world.css_find(lock_css)[index].click()


@step(u'the user "([^"]*)" is enrolled in the course$')
def user_foo_is_enrolled_in_the_course(step, name):
    world.create_user(name, 'test')
    user = User.objects.get(username=name)

    course_id = world.scenario_dict['COURSE'].id
    CourseEnrollment.enroll(user, course_id)


@step(u'Then the asset is (locked|unlocked)$')
def verify_lock_unlock_file(_step, lock_state):
    index = get_index('asset.html')
    assert index != -1, 'Expected to find an asset but could not.'
    lock_css = "input.lock-checkbox"
    checked = world.css_find(lock_css)[index]._element.get_attribute('checked')
    assert_equal(lock_state == "locked", bool(checked))


@step(u'I am at the files and upload page of a Studio course')
def at_upload_page(step):
    step.given('I have opened a new course in studio')
    step.given('I go to the files and uploads page')


@step(u'I have created a course with a (locked|unlocked) asset$')
def open_course_with_locked(step, lock_state):
    step.given('I am at the files and upload page of a Studio course')
    step.given('I upload the file "asset.html"')

    if lock_state == "locked":
        step.given('I lock the asset')
        step.given('I reload the page')


@step(u'Then the asset is (viewable|protected)$')
def view_asset(_step, status):
    url = django_url('/c4x/MITx/999/asset/asset.html')
    if status == 'viewable':
        expected_text = 'test file'
    else:
        expected_text = 'Unauthorized'

    # Note that world.visit would trigger a 403 error instead of displaying "Unauthorized"
    # Instead, we can drop back into the selenium driver get command.
    world.browser.driver.get(url)
    assert_equal(world.css_text('body'), expected_text)


@step('I see a confirmation that the file was deleted$')
def i_see_a_delete_confirmation(_step):
    alert_css = '#notification-confirmation'
    assert world.is_css_present(alert_css)


def get_index(file_name):
    all_names = world.css_find(ASSET_NAMES_CSS)
    for i in range(len(all_names)):
        if file_name == world.css_html(ASSET_NAMES_CSS, index=i):
            return i
    return -1


def get_file(file_name):
    index = get_index(file_name)
    assert index != -1
    url_css = 'a.filename'

    def get_url():
        return world.css_find(url_css)[index]._element.get_attribute('href')
    url = world.retry_on_exception(get_url)
    return requests.get(url)

########NEW FILE########
__FILENAME__ = video
# pylint: disable=C0111

from lettuce import world, step
from nose.tools import assert_less
from xmodule.modulestore import Location
from contentstore.utils import get_modulestore
from selenium.webdriver.common.keys import Keys

VIDEO_BUTTONS = {
    'CC': '.hide-subtitles',
    'volume': '.volume',
    'play': '.video_control.play',
    'pause': '.video_control.pause',
    'handout': '.video-handout.video-download-button a',
}

SELECTORS = {
    'spinner': '.video-wrapper .spinner',
    'controls': 'section.video-controls',
}

# We should wait 300 ms for event handler invocation + 200ms for safety.
DELAY = 0.5


@step('youtube stub server (.*) YouTube API')
def configure_youtube_api(_step, action):
    action=action.strip()
    if action == 'proxies':
        world.youtube.config['youtube_api_blocked'] = False
    elif action == 'blocks':
        world.youtube.config['youtube_api_blocked'] = True
    else:
        raise ValueError('Parameter `action` should be one of "proxies" or "blocks".')


@step('I have created a Video component$')
def i_created_a_video_component(_step):

    world.create_course_with_unit()
    world.create_component_instance(
        step=_step,
        category='video',
    )

    world.wait_for_xmodule()
    world.disable_jquery_animations()

    world.wait_for_present('.is-initialized')
    world.wait(DELAY)
    world.wait_for_invisible(SELECTORS['spinner'])
    if not world.youtube.config.get('youtube_api_blocked'):
        world.wait_for_visible(SELECTORS['controls'])

@step('I have created a Video component with subtitles$')
def i_created_a_video_with_subs(_step):
    _step.given('I have created a Video component with subtitles "OEoXaMPEzfM"')


@step('I have created a Video component with subtitles "([^"]*)"$')
def i_created_a_video_with_subs_with_name(_step, sub_id):
    _step.given('I have created a Video component')

    # Store the current URL so we can return here
    video_url = world.browser.url

    # Upload subtitles for the video using the upload interface
    _step.given('I have uploaded subtitles "{}"'.format(sub_id))

    # Return to the video
    world.visit(video_url)

    world.wait_for_xmodule()

    # update .sub filed with proper subs name (which mimics real Studio/XML behavior)
    # this is needed only for that videos which are created in acceptance tests.
    _step.given('I edit the component')
    world.wait_for_ajax_complete()
    _step.given('I save changes')

    world.disable_jquery_animations()

    world.wait_for_present('.is-initialized')
    world.wait_for_invisible(SELECTORS['spinner'])


@step('I have uploaded subtitles "([^"]*)"$')
def i_have_uploaded_subtitles(_step, sub_id):
    _step.given('I go to the files and uploads page')
    _step.given('I upload the test file "subs_{}.srt.sjson"'.format(sub_id.strip()))


@step('when I view the (.*) it does not have autoplay enabled$')
def does_not_autoplay(_step, video_type):
    world.wait(DELAY)
    world.wait_for_ajax_complete()
    actual = world.css_find('.%s' % video_type)[0]['data-autoplay']
    expected = [u'False', u'false', False]
    assert actual in expected
    assert world.css_has_class('.video_control', 'play')


@step('creating a video takes a single click$')
def video_takes_a_single_click(_step):
    component_css = '.xmodule_VideoModule'
    assert world.is_css_not_present(component_css)

    world.css_click("a[data-category='video']")
    assert world.is_css_present(component_css)


@step('I edit the component$')
def i_edit_the_component(_step):
    world.edit_component()


@step('I have (hidden|toggled) captions$')
def hide_or_show_captions(step, shown):
    button_css = 'a.hide-subtitles'
    if shown == 'hidden':
        world.css_click(button_css)
    if shown == 'toggled':
        world.css_click(button_css)
        # When we click the first time, a tooltip shows up. We want to
        # click the button rather than the tooltip, so move the mouse
        # away to make it disappear.
        button = world.css_find(button_css)
        # mouse_out is not implemented on firefox with selenium
        if not world.is_firefox:
            button.mouse_out()
        world.css_click(button_css)


@step('I have created a video with only XML data$')
def xml_only_video(step):
    # Create a new video *without* metadata. This requires a certain
    # amount of rummaging to make sure all the correct data is present
    step.given('I have clicked the new unit button')

    # Wait for the new unit to be created and to load the page
    world.wait(1)

    course = world.scenario_dict['COURSE']
    store = get_modulestore(course.location)

    parent_location = store.get_items(course.id, category='vertical', revision='draft')[0].location

    youtube_id = 'ABCDEFG'
    world.scenario_dict['YOUTUBE_ID'] = youtube_id

    # Create a new Video component, but ensure that it doesn't have
    # metadata. This allows us to test that we are correctly parsing
    # out XML
    world.ItemFactory.create(
        parent_location=parent_location,
        category='video',
        data='<video youtube="1.00:%s"></video>' % youtube_id,
        modulestore=store,
    )


@step('The correct Youtube video is shown$')
def the_youtube_video_is_shown(_step):
    ele = world.css_find('.video').first
    assert ele['data-streams'].split(':')[1] == world.scenario_dict['YOUTUBE_ID']


@step('Make sure captions are (.+)$')
def set_captions_visibility_state(_step, captions_state):
    SELECTOR = '.closed .subtitles'
    world.wait_for_visible('.hide-subtitles')
    if captions_state == 'closed':
        if not world.is_css_present(SELECTOR):
            world.css_find('.hide-subtitles').click()
    else:
        if world.is_css_present(SELECTOR):
            world.css_find('.hide-subtitles').click()


@step('I hover over button "([^"]*)"$')
def hover_over_button(_step, button):
    world.css_find(VIDEO_BUTTONS[button.strip()]).mouse_over()


@step('Captions (?:are|become) "([^"]*)"$')
def check_captions_visibility_state(_step, visibility_state):
    if visibility_state == 'visible':
        assert world.css_visible('.subtitles')
    else:
        assert not world.css_visible('.subtitles')


def find_caption_line_by_data_index(index):
    SELECTOR = ".subtitles > li[data-index='{index}']".format(index=index)
    return world.css_find(SELECTOR).first


@step('I focus on caption line with data-index "([^"]*)"$')
def focus_on_caption_line(_step, index):
    world.wait_for_present('.video.is-captions-rendered')
    world.wait_for(lambda _: world.css_text('.subtitles'), timeout=30)
    find_caption_line_by_data_index(int(index.strip()))._element.send_keys(Keys.TAB)


@step('I press "enter" button on caption line with data-index "([^"]*)"$')
def click_on_the_caption(_step, index):
    world.wait_for_present('.video.is-captions-rendered')
    world.wait_for(lambda _: world.css_text('.subtitles'), timeout=30)
    find_caption_line_by_data_index(int(index.strip()))._element.send_keys(Keys.ENTER)


@step('I see caption line with data-index "([^"]*)" has class "([^"]*)"$')
def caption_line_has_class(_step, index, className):
    SELECTOR = ".subtitles > li[data-index='{index}']".format(index=int(index.strip()))
    assert world.css_has_class(SELECTOR, className.strip())


@step('I see a range on slider$')
def see_a_range_slider_with_proper_range(_step):
    world.wait_for_visible(VIDEO_BUTTONS['pause'])
    assert world.css_visible(".slider-range")


@step('I (.*) see video button "([^"]*)"$')
def do_not_see_or_not_button_video(_step, action, button_type):
    world.wait(DELAY)
    world.wait_for_ajax_complete()
    action=action.strip()
    button = button_type.strip()
    if action == 'do not':
        assert not world.is_css_present(VIDEO_BUTTONS[button])
    elif action == 'can':
        assert world.css_visible(VIDEO_BUTTONS[button])
    else:
        raise ValueError('Parameter `action` should be one of "do not" or "can".')


@step('I click video button "([^"]*)"$')
def click_button_video(_step, button_type):
    world.wait(DELAY)
    world.wait_for_ajax_complete()
    button = button_type.strip()
    world.css_click(VIDEO_BUTTONS[button])


@step('I seek video to "([^"]*)" seconds$')
def seek_video_to_n_seconds(_step, seconds):
    time = float(seconds.strip())
    jsCode = "$('.video').data('video-player-state').videoPlayer.onSlideSeek({{time: {0:f}}})".format(time)
    world.browser.execute_script(jsCode)


@step('I see video starts playing from "([^"]*)" position$')
def start_playing_video_from_n_seconds(_step, position):
    world.wait_for(
        func=lambda _: world.css_html('.vidtime')[:4] == position.strip(),
        timeout=5
    )

########NEW FILE########
__FILENAME__ = video_editor
# -*- coding: utf-8 -*-
# disable missing docstring
# pylint: disable=C0111

import requests
from lettuce import world, step
from nose.tools import assert_true, assert_equal, assert_in, assert_not_equal # pylint: disable=E0611
from terrain.steps import reload_the_page
from django.conf import settings
from common import upload_file, attach_file

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT

DISPLAY_NAME = "Component Display Name"
NATIVE_LANGUAGES = {lang: label for lang, label in settings.LANGUAGES if len(lang) == 2}
LANGUAGES = {
    lang: NATIVE_LANGUAGES.get(lang, display)
    for lang, display in settings.ALL_LANGUAGES
}

TRANSLATION_BUTTONS = {
    'add': '.metadata-video-translations .create-action',
    'upload': '.metadata-video-translations .upload-action',
    'download': '.metadata-video-translations .download-action',
    'remove': '.metadata-video-translations .remove-action',
    'clear': '.metadata-video-translations .setting-clear',
}

VIDEO_MENUS = {
    'language': '.lang .menu',
}


class RequestHandlerWithSessionId(object):
    def get(self, url):
        """
        Sends a request.
        """
        kwargs = dict()

        session_id = [{i['name']:i['value']} for i in world.browser.cookies.all() if i['name'] == u'sessionid']
        if session_id:
            kwargs.update({
                'cookies': session_id[0]
            })

        response = requests.get(url, **kwargs)
        self.response = response
        self.status_code = response.status_code
        self.headers = response.headers
        self.content = response.content

        return self

    def is_success(self):
        """
        Returns `True` if the response was succeed, otherwise, returns `False`.
        """
        if self.status_code < 400:
            return True
        return False

    def check_header(self, name, value):
        """
        Returns `True` if the response header exist and has appropriate value,
        otherwise, returns `False`.
        """
        if value in self.headers.get(name, ''):
            return True
        return False


def success_upload_file(filename):
    upload_file(filename, sub_path="uploads/")
    world.css_has_text('#upload_confirm', 'Success!')
    world.is_css_not_present('.wrapper-modal-window-assetupload', wait_time=30)


def get_translations_container():
    return world.browser.find_by_xpath('//label[text()="Transcript Languages"]/following-sibling::div')


def get_setting_container(lang_code):
    try:
        get_xpath = lambda value: './/descendant::a[@data-lang="{}" and contains(@class,"remove-setting")]/parent::*'.format(value)
        return get_translations_container().find_by_xpath(get_xpath(lang_code)).first
    except Exception:
        return None


def get_last_dropdown():
    return get_translations_container().find_by_xpath('.//descendant::select[last()]').last


def choose_option(dropdown, value):
    dropdown.find_by_value(value)[0].click()


def choose_new_lang(lang_code):
    world.css_click(TRANSLATION_BUTTONS['add'])
    choose_option(get_last_dropdown(), lang_code)
    assert_equal(get_last_dropdown().value, lang_code, "Option with provided value is not available or was not selected")


def open_menu(menu):
    world.browser.execute_script("$('{selector}').parent().addClass('is-opened')".format(
        selector=VIDEO_MENUS[menu]
    ))


@step('I have set "transcript display" to (.*)$')
def set_show_captions(step, setting):
    # Prevent cookies from overriding course settings
    world.browser.cookies.delete('hide_captions')

    world.edit_component()
    world.select_editor_tab('Advanced')
    world.browser.select('Show Transcript', setting)
    world.save_component()


@step('when I view the video it (.*) show the captions$')
def shows_captions(_step, show_captions):
    world.wait_for_js_variable_truthy("Video")
    world.wait(0.5)
    if show_captions == 'does not':
        assert_true(world.is_css_present('div.video.closed'))
    else:
        assert_true(world.is_css_not_present('div.video.closed'))

    # Prevent cookies from overriding course settings
    world.browser.cookies.delete('hide_captions')
    world.browser.cookies.delete('current_player_mode')


@step('I see the correct video settings and default values$')
def correct_video_settings(_step):
    expected_entries = [
        # basic
        [DISPLAY_NAME, 'Video', False],
        ['Default Video URL', 'http://youtu.be/OEoXaMPEzfM, , ', False],

        # advanced
        [DISPLAY_NAME, 'Video', False],
        ['Default Timed Transcript', '', False],
        ['Download Transcript Allowed', 'False', False],
        ['Downloadable Transcript URL', '', False],
        ['Show Transcript', 'True', False],
        ['Transcript Languages', '', False],
        ['Upload Handout', '', False],
        ['Video Download Allowed', 'False', False],
        ['Video File URLs', '', False],
        ['Video Start Time', '00:00:00', False],
        ['Video Stop Time', '00:00:00', False],
        ['YouTube ID', 'OEoXaMPEzfM', False],
        ['YouTube ID for .75x speed', '', False],
        ['YouTube ID for 1.25x speed', '', False],
        ['YouTube ID for 1.5x speed', '', False]
    ]
    world.verify_all_setting_entries(expected_entries)


@step('my video display name change is persisted on save$')
def video_name_persisted(step):
    world.save_component()
    reload_the_page(step)
    world.wait_for_xmodule()
    world.edit_component()

    world.verify_setting_entry(
        world.get_setting_entry(DISPLAY_NAME),
        DISPLAY_NAME, '3.4', True
    )


@step('I can modify video display name')
def i_can_modify_video_display_name(_step):
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, '3.4')
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, '3.4', True)


@step('I upload transcript file(?:s)?:$')
def upload_transcript(step):
    input_hidden = '.metadata-video-translations .input'
    # Number of previously added translations
    initial_index = len(world.css_find(TRANSLATION_BUTTONS['download']))

    if step.hashes:
        for i, item in enumerate(step.hashes):
            lang_code = item['lang_code']
            filename = item['filename']
            index = initial_index + i

            choose_new_lang(lang_code)

            expected_text = world.css_text(TRANSLATION_BUTTONS['upload'], index=index)
            assert_equal(expected_text, "Upload")
            assert_equal(world.css_find(input_hidden).last.value, "")

            world.css_click(TRANSLATION_BUTTONS['upload'], index=index)
            success_upload_file(filename)

            world.wait_for_visible(TRANSLATION_BUTTONS['download'], index=index)
            assert_equal(world.css_find(TRANSLATION_BUTTONS['upload']).last.text, "Replace")
            assert_equal(world.css_find(input_hidden).last.value, filename)


@step('I try to upload transcript file "([^"]*)"$')
def try_to_upload_transcript(step, filename):
    world.css_click(TRANSLATION_BUTTONS['upload'])
    attach_file(filename, 'uploads/')


@step('I upload transcript file "([^"]*)" for "([^"]*)" language code$')
def upload_transcript_for_lang(step, filename, lang_code):
    get_xpath = lambda value: './/div/a[contains(@class, "upload-action")]'.format(value)
    container = get_setting_container(lang_code)

    # If translation isn't uploaded, prepare drop-down and try to find container again
    choose_new_lang(lang_code)
    container = get_setting_container(lang_code)

    button = container.find_by_xpath(get_xpath(lang_code)).first
    button.click()
    success_upload_file(filename)


@step('I replace transcript file for "([^"]*)" language code by "([^"]*)"$')
def replace_transcript_for_lang(step, lang_code, filename):
    get_xpath = lambda value: './/div/a[contains(@class, "upload-action")]'.format(value)
    container = get_setting_container(lang_code)

    button = container.find_by_xpath(get_xpath(lang_code)).first
    button.click()
    success_upload_file(filename)


@step('I see validation error "([^"]*)"$')
def verify_validation_error_message(step, error_message):
    assert_equal(world.css_text('#upload_error'), error_message)


@step('I can download transcript for "([^"]*)" language code, that contains text "([^"]*)"$')
def i_can_download_transcript(_step, lang_code, text):
    MIME_TYPE = 'application/x-subrip'
    get_xpath = lambda value: './/div/a[contains(text(), "Download")]'.format(value)
    container = get_setting_container(lang_code)
    assert container
    button = container.find_by_xpath(get_xpath(lang_code)).first
    url = button['href']
    request = RequestHandlerWithSessionId()
    assert_true(request.get(url).is_success())
    assert_true(request.check_header('content-type', MIME_TYPE))
    assert_in(text.encode('utf-8'), request.content)


@step('I remove translation for "([^"]*)" language code$')
def i_can_remove_transcript(_step, lang_code):
    get_xpath = lambda value: './/descendant::a[@data-lang="{}" and contains(@class,"remove-setting")]'.format(value)
    container = get_setting_container(lang_code)
    assert container
    button = container.find_by_xpath(get_xpath(lang_code)).first
    button.click()


@step('I see translations for "([^"]*)"$')
def verify_translations(_step, lang_codes_string):
    expected = [l.strip() for l in lang_codes_string.split(',')]
    actual = [l['data-lang'] for l in world.css_find('.metadata-video-translations .remove-setting')]
    assert_equal(set(expected), set(actual))


@step('I do not see translations$')
def no_translations(_step):
    assert_true(world.is_css_not_present('.metadata-video-translations .remove-setting'))


@step('I confirm prompt$')
def confirm_prompt(_step):
    world.confirm_studio_prompt()


@step('I (cannot )?choose "([^"]*)" language code$')
def i_choose_lang_code(_step, cannot, lang_code):
    choose_option(get_last_dropdown(), lang_code)
    if cannot:
        assert_not_equal(get_last_dropdown().value, lang_code, "Option with provided value was selected, but shouldn't")
    else:
        assert_equal(get_last_dropdown().value, lang_code, "Option with provided value is not available or was not selected")


@step('I click button "([^"]*)"$')
def click_button(_step, button):
    world.css_click(TRANSLATION_BUTTONS[button.lower()])


@step('video language menu has "([^"]*)" translations$')
def i_see_correct_langs(_step, langs):
    menu_name = 'language'
    open_menu(menu_name)
    items = world.css_find(VIDEO_MENUS[menu_name] + ' li')
    translations = {t.strip(): LANGUAGES[t.strip()] for t in langs.split(',')}

    assert_equal(len(translations), len(items))
    for lang_code, label in translations.items():
        assert_true(any([i.text == label for i in items]))
        assert_true(any([i['data-lang-code'] == lang_code for i in items]))

########NEW FILE########
__FILENAME__ = video_handout
# -*- coding: utf-8 -*-
# disable missing docstring
# pylint: disable=C0111

from lettuce import world, step
from nose.tools import assert_true # pylint: disable=E0611
from video_editor import RequestHandlerWithSessionId, success_upload_file


@step('I (?:upload|replace) handout file(?: by)? "([^"]*)"$')
def upload_handout(step, filename):
    world.css_click('.wrapper-comp-setting.file-uploader .upload-action')
    success_upload_file(filename)


@step('I can download handout file( in editor)? with mime type "([^"]*)"$')
def i_can_download_handout_with_mime_type(_step, is_editor, mime_type):
    if is_editor:
        selector = '.wrapper-comp-setting.file-uploader .download-action'
    else:
        selector = '.video-handout.video-download-button a'

    button = world.css_find(selector).first
    url = button['href']
    request = RequestHandlerWithSessionId()
    assert_true(request.get(url).is_success())
    assert_true(request.check_header('content-type', mime_type))


@step('I clear handout$')
def clear_handout(_step):
    world.css_click('.wrapper-comp-setting.file-uploader .setting-clear')


@step('I have created a Video component with handout file "([^"]*)"')
def create_video_with_handout(_step, filename):
    _step.given('I have created a Video component')
    _step.given('I edit the component')
    _step.given('I open tab "Advanced"')
    _step.given('I upload handout file "{0}"'.format(filename))

########NEW FILE########
__FILENAME__ = git_export_utils
"""
Utilities for export a course's XML into a git repository,
committing and pushing the changes.
"""

import logging
import os
import subprocess
from urlparse import urlparse

from django.conf import settings
from django.contrib.auth.models import User
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_exporter import export_to_xml

log = logging.getLogger(__name__)

GIT_REPO_EXPORT_DIR = getattr(settings, 'GIT_REPO_EXPORT_DIR', None)
GIT_EXPORT_DEFAULT_IDENT = getattr(settings, 'GIT_EXPORT_DEFAULT_IDENT',
                                   {'name': 'STUDIO_EXPORT_TO_GIT',
                                    'email': 'STUDIO_EXPORT_TO_GIT@example.com'})


class GitExportError(Exception):
    """
    Convenience exception class for git export error conditions.
    """

    NO_EXPORT_DIR = _("GIT_REPO_EXPORT_DIR not set or path {0} doesn't exist, "
                      "please create it, or configure a different path with "
                      "GIT_REPO_EXPORT_DIR".format(GIT_REPO_EXPORT_DIR))
    URL_BAD = _('Non writable git url provided. Expecting something like:'
                ' git@github.com:mitocw/edx4edx_lite.git')
    URL_NO_AUTH = _('If using http urls, you must provide the username '
                    'and password in the url. Similar to '
                    'https://user:pass@github.com/user/course.')
    DETACHED_HEAD = _('Unable to determine branch, repo in detached HEAD mode')
    CANNOT_PULL = _('Unable to update or clone git repository.')
    XML_EXPORT_FAIL = _('Unable to export course to xml.')
    CONFIG_ERROR = _('Unable to configure git username and password')
    CANNOT_COMMIT = _('Unable to commit changes. This is usually '
                      'because there are no changes to be committed')
    CANNOT_PUSH = _('Unable to push changes.  This is usually '
                    'because the remote repository cannot be contacted')
    BAD_COURSE = _('Bad course location provided')
    MISSING_BRANCH = _('Missing branch on fresh clone')


def cmd_log(cmd, cwd):
    """
    Helper function to redirect stderr to stdout and log the command
    used along with the output. Will raise subprocess.CalledProcessError if
    command doesn't return 0, and returns the command's output.
    """
    output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT)
    log.debug(_('Command was: {0!r}. '
                'Working directory was: {1!r}'.format(' '.join(cmd), cwd)))
    log.debug(_('Command output was: {0!r}'.format(output)))
    return output


def export_to_git(course_id, repo, user='', rdir=None):
    """Export a course to git."""
    # pylint: disable=R0915

    if not GIT_REPO_EXPORT_DIR:
        raise GitExportError(GitExportError.NO_EXPORT_DIR)

    if not os.path.isdir(GIT_REPO_EXPORT_DIR):
        raise GitExportError(GitExportError.NO_EXPORT_DIR)

    # Check for valid writable git url
    if not (repo.endswith('.git') or
            repo.startswith(('http:', 'https:', 'file:'))):
        raise GitExportError(GitExportError.URL_BAD)

    # Check for username and password if using http[s]
    if repo.startswith('http:') or repo.startswith('https:'):
        parsed = urlparse(repo)
        if parsed.username is None or parsed.password is None:
            raise GitExportError(GitExportError.URL_NO_AUTH)
    if rdir:
        rdir = os.path.basename(rdir)
    else:
        rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]

    log.debug("rdir = %s", rdir)

    # Pull or clone repo before exporting to xml
    # and update url in case origin changed.
    rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir)
    branch = None
    if os.path.exists(rdirp):
        log.info(_('Directory already exists, doing a git reset and pull '
                   'instead of git clone.'))
        cwd = rdirp
        # Get current branch
        cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
        try:
            branch = cmd_log(cmd, cwd).strip('\n')
        except subprocess.CalledProcessError as ex:
            log.exception('Failed to get branch: %r', ex.output)
            raise GitExportError(GitExportError.DETACHED_HEAD)

        cmds = [
            ['git', 'remote', 'set-url', 'origin', repo],
            ['git', 'fetch', 'origin'],
            ['git', 'reset', '--hard', 'origin/{0}'.format(branch)],
            ['git', 'pull'],
        ]
    else:
        cmds = [['git', 'clone', repo]]
        cwd = GIT_REPO_EXPORT_DIR

    cwd = os.path.abspath(cwd)
    for cmd in cmds:
        try:
            cmd_log(cmd, cwd)
        except subprocess.CalledProcessError as ex:
            log.exception('Failed to pull git repository: %r', ex.output)
            raise GitExportError(GitExportError.CANNOT_PULL)

    # export course as xml before commiting and pushing
    root_dir = os.path.dirname(rdirp)
    course_dir = os.path.splitext(os.path.basename(rdirp))[0]
    try:
        export_to_xml(modulestore('direct'), contentstore(), course_id,
                      root_dir, course_dir, modulestore())
    except (EnvironmentError, AttributeError):
        log.exception('Failed export to xml')
        raise GitExportError(GitExportError.XML_EXPORT_FAIL)

    # Get current branch if not already set
    if not branch:
        cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
        try:
            branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n')
        except subprocess.CalledProcessError as ex:
            log.exception('Failed to get branch from freshly cloned repo: %r',
                          ex.output)
            raise GitExportError(GitExportError.MISSING_BRANCH)

    # Now that we have fresh xml exported, set identity, add
    # everything to git, commit, and push to the right branch.
    ident = {}
    try:
        user = User.objects.get(username=user)
        ident['name'] = user.username
        ident['email'] = user.email
    except User.DoesNotExist:
        # That's ok, just use default ident
        ident = GIT_EXPORT_DEFAULT_IDENT
    time_stamp = timezone.now()
    cwd = os.path.abspath(rdirp)
    commit_msg = 'Export from Studio at {1}'.format(user, time_stamp)
    try:
        cmd_log(['git', 'config', 'user.email', ident['email']], cwd)
        cmd_log(['git', 'config', 'user.name', ident['name']], cwd)
    except subprocess.CalledProcessError as ex:
        log.exception('Error running git configure commands: %r', ex.output)
        raise GitExportError(GitExportError.CONFIG_ERROR)
    try:
        cmd_log(['git', 'add', '.'], cwd)
        cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd)
    except subprocess.CalledProcessError as ex:
        log.exception('Unable to commit changes: %r', ex.output)
        raise GitExportError(GitExportError.CANNOT_COMMIT)
    try:
        cmd_log(['git', 'push', '-q', 'origin', branch], cwd)
    except subprocess.CalledProcessError as ex:
        log.exception('Error running git push command: %r', ex.output)
        raise GitExportError(GitExportError.CANNOT_PUSH)

########NEW FILE########
__FILENAME__ = check_course
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import check_module_metadata_editability
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    help = '''Enumerates through the course and find common errors'''

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError("check_course requires one argument: <course_id>")

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        store = modulestore()

        course = store.get_course(course_key, depth=3)

        err_cnt = 0

        def _xlint_metadata(module):
            err_cnt = check_module_metadata_editability(module)
            for child in module.get_children():
                err_cnt = err_cnt + _xlint_metadata(child)
            return err_cnt

        err_cnt = err_cnt + _xlint_metadata(course)

        # we've had a bug where the xml_attributes field can we rewritten as a string rather than a dict
        def _check_xml_attributes_field(module):
            err_cnt = 0
            if hasattr(module, 'xml_attributes') and isinstance(module.xml_attributes, basestring):
                print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location)
                err_cnt = err_cnt + 1
            for child in module.get_children():
                err_cnt = err_cnt + _check_xml_attributes_field(child)
            return err_cnt

        err_cnt = err_cnt + _check_xml_attributes_field(course)

        # check for dangling discussion items, this can cause errors in the forums
        def _get_discussion_items(module):
            discussion_items = []
            if module.location.category == 'discussion':
                discussion_items = discussion_items + [module.location]

            for child in module.get_children():
                discussion_items = discussion_items + _get_discussion_items(child)

            return discussion_items

        discussion_items = _get_discussion_items(course)

        # now query all discussion items via get_items() and compare with the tree-traversal
        queried_discussion_items = store.get_items(course_key=course_key, category='discussion',)

        for item in queried_discussion_items:
            if item.location not in discussion_items:
                print 'Found dangling discussion module = {0}'.format(item.location)

########NEW FILE########
__FILENAME__ = clone_course
"""
Script for cloning a course
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.store_utilities import clone_course
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from student.roles import CourseInstructorRole, CourseStaffRole
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


#
# To run from command line: ./manage.py cms clone_course --settings=dev master/300/cough edx/111/foo
#
class Command(BaseCommand):
    """Clone a MongoDB-backed course to another location"""
    help = 'Clone a MongoDB backed course to another location'

    def course_key_from_arg(self, arg):
        """
        Convert the command line arg into a course key
        """
        try:
            return CourseKey.from_string(arg)
        except InvalidKeyError:
            return SlashSeparatedCourseKey.from_deprecated_string(arg)

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 2:
            raise CommandError("clone requires 2 arguments: <source-course_id> <dest-course_id>")

        source_course_id = self.course_key_from_arg(args[0])
        dest_course_id = self.course_key_from_arg(args[1])

        mstore = modulestore('direct')
        cstore = contentstore()

        mstore.ignore_write_events_on_courses.add(dest_course_id)

        print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))

        if clone_course(mstore, cstore, source_course_id, dest_course_id):
            print("copying User permissions...")
            # purposely avoids auth.add_user b/c it doesn't have a caller to authorize
            CourseInstructorRole(dest_course_id).add_users(
                *CourseInstructorRole(source_course_id).users_with_role()
            )
            CourseStaffRole(dest_course_id).add_users(
                *CourseStaffRole(source_course_id).users_with_role()
            )

########NEW FILE########
__FILENAME__ = course_id_clash
"""
Script for finding all courses whose org/name pairs == other courses when ignoring case
"""
from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore


#
# To run from command line: ./manage.py cms --settings dev course_id_clash
#
class Command(BaseCommand):
    """
    Script for finding all courses whose org/name pairs == other courses when ignoring case
    """
    help = 'List all courses ids which may collide when ignoring case'

    def handle(self, *args, **options):
        mstore = modulestore()
        if hasattr(mstore, 'collection'):
            map_fn = '''
                function () {
                    emit(this._id.org.toLowerCase()+this._id.course.toLowerCase(), {target: this._id});
                }
            '''
            reduce_fn = '''
                function (idpair, matches) {
                    var result = {target: []};
                    matches.forEach(function (match) {
                        result.target.push(match.target);
                    });
                    return result;
                }
            '''
            finalize = '''
                function(key, reduced) {
                    if (Array.isArray(reduced.target)) {
                        return reduced;
                    }
                    else {return null;}
                    }
            '''
            results = mstore.collection.map_reduce(
                map_fn, reduce_fn, {'inline': True}, query={'_id.category': 'course'}, finalize=finalize
            )
            results = results.get('results')
            for entry in results:
                if entry.get('value') is not None:
                    print '{:-^40}'.format(entry.get('_id'))
                    for course_id in entry.get('value').get('target'):
                        print '   {}/{}/{}'.format(course_id.get('org'), course_id.get('course'), course_id.get('name'))


########NEW FILE########
__FILENAME__ = delete_course
###
### Script for cloning a course
###
from django.core.management.base import BaseCommand, CommandError
from .prompt import query_yes_no
from contentstore.utils import delete_course_and_groups
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    help = '''Delete a MongoDB backed course'''

    def handle(self, *args, **options):
        if len(args) != 1 and len(args) != 2:
            raise CommandError("delete_course requires one or more arguments: <course_id> |commit|")

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        commit = False
        if len(args) == 2:
            commit = args[1] == 'commit'

        if commit:
            print('Actually going to delete the course from DB....')

        if query_yes_no("Deleting course {0}. Confirm?".format(course_key), default="no"):
            if query_yes_no("Are you sure. This action cannot be undone!", default="no"):
                delete_course_and_groups(course_key, commit)

########NEW FILE########
__FILENAME__ = edit_course_tabs
###
### Script for editing the course's tabs
###

#
# Run it this way:
#   ./manage.py cms --settings dev edit_course_tabs --course Stanford/CS99/2013_spring
#
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from .prompt import query_yes_no

from courseware.courses import get_course_by_id

from contentstore.views import tabs
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.keys import CourseKey


def print_course(course):
    "Prints out the course id and a numbered list of tabs."
    print course.id
    print 'num type name'
    for index, item in enumerate(course.tabs):
        print index + 1, '"' + item.get('type') + '"', '"' + item.get('name', '') + '"'


# course.tabs looks like this
# [{u'type': u'courseware'}, {u'type': u'course_info', u'name': u'Course Info'}, {u'type': u'textbooks'},
# {u'type': u'discussion', u'name': u'Discussion'}, {u'type': u'wiki', u'name': u'Wiki'},
# {u'type': u'progress', u'name': u'Progress'}]


class Command(BaseCommand):
    help = """See and edit a course's tabs list. Only supports insertion
and deletion. Move and rename etc. can be done with a delete
followed by an insert. The tabs are numbered starting with 1.
Tabs 1 and 2 cannot be changed, and tabs of type static_tab cannot
be edited (use Studio for those).

As a first step, run the command with a courseid like this:
  --course Stanford/CS99/2013_spring
This will print the existing tabs types and names. Then run the
command again, adding --insert or --delete to edit the list.
"""
    # Making these option objects separately, so can refer to their .help below
    course_option = make_option('--course',
                                action='store',
                                dest='course',
                                default=False,
                                help='--course <id> required, e.g. Stanford/CS99/2013_spring')
    delete_option = make_option('--delete',
                                action='store_true',
                                dest='delete',
                                default=False,
                                help='--delete <tab-number>')
    insert_option = make_option('--insert',
                                action='store_true',
                                dest='insert',
                                default=False,
                                help='--insert <tab-number> <type> <name>, e.g. 2 "course_info" "Course Info"')

    option_list = BaseCommand.option_list + (course_option, delete_option, insert_option)

    def handle(self, *args, **options):
        if not options['course']:
            raise CommandError(Command.course_option.help)

        try:
            course_key = CourseKey.from_string(options['course'])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(options['course'])

        course = get_course_by_id(course_key)

        print 'Warning: this command directly edits the list of course tabs in mongo.'
        print 'Tabs before any changes:'
        print_course(course)

        try:
            if options['delete']:
                if len(args) != 1:
                    raise CommandError(Command.delete_option.help)
                num = int(args[0])
                if query_yes_no('Deleting tab {0} Confirm?'.format(num), default='no'):
                    tabs.primitive_delete(course, num - 1)  # -1 for 0-based indexing
            elif options['insert']:
                if len(args) != 3:
                    raise CommandError(Command.insert_option.help)
                num = int(args[0])
                tab_type = args[1]
                name = args[2]
                if query_yes_no('Inserting tab {0} "{1}" "{2}" Confirm?'.format(num, tab_type, name), default='no'):
                    tabs.primitive_insert(course, num - 1, tab_type, name)  # -1 as above
        except ValueError as e:
            # Cute: translate to CommandError so the CLI error prints nicely.
            raise CommandError(e)

########NEW FILE########
__FILENAME__ = empty_asset_trashcan
from django.core.management.base import BaseCommand, CommandError
from xmodule.contentstore.utils import empty_asset_trashcan
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from .prompt import query_yes_no
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    help = '''Empty the trashcan. Can pass an optional course_id to limit the damage.'''

    def handle(self, *args, **options):
        if len(args) != 1 and len(args) != 0:
            raise CommandError("empty_asset_trashcan requires one or no arguments: |<course_id>|")

        if len(args) == 1:
            try:
                course_key = CourseKey.from_string(args[0])
            except InvalidKeyError:
                course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

            course_ids = [course_key]
        else:
            course_ids = [course.id for course in modulestore('direct').get_courses()]

        if query_yes_no("Emptying trashcan. Confirm?", default="no"):
            empty_asset_trashcan(course_ids)

########NEW FILE########
__FILENAME__ = export
"""
Script for exporting courseware from Mongo to a tar.gz file
"""
import os

from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.contentstore.django import contentstore
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    """
    Export the specified data directory into the default ModuleStore
    """
    help = 'Export the specified data directory into the default ModuleStore'

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 2:
            raise CommandError("export requires two arguments: <course id> <output path>")

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        output_path = args[1]

        print("Exporting course id = {0} to {1}".format(course_key, output_path))

        root_dir = os.path.dirname(output_path)
        course_dir = os.path.splitext(os.path.basename(output_path))[0]

        export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())

########NEW FILE########
__FILENAME__ = export_all_courses
"""
Script for exporting all courseware from Mongo to a directory
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore


class Command(BaseCommand):
    """Export all courses from mongo to the specified data directory"""
    help = 'Export all courses from mongo to the specified data directory'

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 1:
            raise CommandError("export requires one argument: <output path>")

        output_path = args[0]

        cs = contentstore()
        ms = modulestore('direct')
        root_dir = output_path
        courses = ms.get_courses()

        print("%d courses to export:" % len(courses))
        cids = [x.id for x in courses]
        print(cids)

        for course_id in cids:

            print("-"*77)
            print("Exporting course id = {0} to {1}".format(course_id, output_path))

            if 1:
                try:
                    course_dir = course_id.replace('/', '...')
                    export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
                except Exception as err:
                    print("="*30 + "> Oops, failed to export %s" % course_id)
                    print("Error:")
                    print(err)

########NEW FILE########
__FILENAME__ = export_convert_format
"""
Script for converting a tar.gz file representing an exported course
to the archive format used by a different version of export.

Sample invocation: ./manage.py export_convert_format mycourse.tar.gz ~/newformat/
"""
import os
from path import path
from django.core.management.base import BaseCommand, CommandError

from tempfile import mkdtemp
import tarfile
import shutil
from extract_tar import safetar_extractall

from xmodule.modulestore.xml_exporter import convert_between_versions


class Command(BaseCommand):
    """
    Convert between export formats.
    """
    help = 'Convert between versions 0 and 1 of the course export format'
    args = '<tar.gz archive file> <output path>'

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 2:
            raise CommandError("export requires two arguments: <tar.gz file> <output path>")

        source_archive = args[0]
        output_path = args[1]

        # Create temp directories to extract the source and create the target archive.
        temp_source_dir = mkdtemp()
        temp_target_dir = mkdtemp()
        try:
            extract_source(source_archive, temp_source_dir)

            desired_version = convert_between_versions(temp_source_dir, temp_target_dir)

            # New zip up the target directory.
            parts = os.path.basename(source_archive).split('.')
            archive_name = path(output_path) / "{source_name}_version_{desired_version}.tar.gz".format(
                source_name=parts[0], desired_version=desired_version
            )
            with open(archive_name, "w"):
                tar_file = tarfile.open(archive_name, mode='w:gz')
                try:
                    for item in os.listdir(temp_target_dir):
                        tar_file.add(path(temp_target_dir) / item, arcname=item)

                finally:
                    tar_file.close()

            print("Created archive {0}".format(archive_name))

        except ValueError as err:
            raise CommandError(err)

        finally:
            shutil.rmtree(temp_source_dir)
            shutil.rmtree(temp_target_dir)


def extract_source(source_archive, target):
    """
    Extract the archive into the given target directory.
    """
    with tarfile.open(source_archive) as tar_file:
        safetar_extractall(tar_file, target)

########NEW FILE########
__FILENAME__ = git_export
"""
This command exports a course from CMS to a git repository.
It takes as arguments the course id to export (i.e MITx/999/2020 ) and
the repository to commit too.  It takes username as an option for identifying
the commit, as well as a directory path to place the git repository.

By default it will use settings.GIT_REPO_EXPORT_DIR/repo_name as the cloned
directory.  It is branch aware, but will reset all local changes to the
repository before attempting to export the XML, add, and commit changes if
any have taken place.

This functionality is also available as an export view in studio if the giturl
attribute is set and the FEATURE['ENABLE_EXPORT_GIT'] is set.
"""

import logging
from optparse import make_option

from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import ugettext as _

import contentstore.git_export_utils as git_export_utils
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
from contentstore.git_export_utils import GitExportError
from xmodule.modulestore.keys import CourseKey

log = logging.getLogger(__name__)


class Command(BaseCommand):
    """
    Take a course from studio and export it to a git repository.
    """

    option_list = BaseCommand.option_list + (
        make_option('--username', '-u', dest='user',
                    help=('Specify a username from LMS/Studio to be used '
                          'as the commit author.')),
        make_option('--repo_dir', '-r', dest='repo',
                    help='Specify existing git repo directory.'),
    )

    help = _('Take the specified course and attempt to '
             'export it to a git repository\n. Course directory '
             'must already be a git repository. Usage: '
             ' git_export <course_loc> <git_url>')

    def handle(self, *args, **options):
        """
        Checks arguments and runs export function if they are good
        """

        if len(args) != 2:
            raise CommandError('This script requires exactly two arguments: '
                               'course_loc and git_url')

        # Rethrow GitExportError as CommandError for SystemExit
        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            try:
                course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
            except InvalidKeyError:
                raise CommandError(GitExportError.BAD_COURSE)

        try:
            git_export_utils.export_to_git(
                course_key,
                args[1],
                options.get('user', ''),
                options.get('rdir', None)
            )
        except git_export_utils.GitExportError as ex:
            raise CommandError(str(ex))

########NEW FILE########
__FILENAME__ = import
"""
Script for importing courseware from XML format
"""

from django.core.management.base import BaseCommand, CommandError, make_option
from django_comment_common.utils import (seed_permissions_roles,
                                         are_permissions_roles_seeded)
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore


class Command(BaseCommand):
    """
    Import the specified data directory into the default ModuleStore
    """
    help = 'Import the specified data directory into the default ModuleStore'

    option_list = BaseCommand.option_list + (
        make_option('--nostatic',
                    action='store_true',
                    help='Skip import of static content'),
    )

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) == 0:
            raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")

        data_dir = args[0]
        do_import_static = not (options.get('nostatic', False))
        if len(args) > 1:
            course_dirs = args[1:]
        else:
            course_dirs = None
        self.stdout.write("Importing.  Data_dir={data}, course_dirs={courses}\n".format(
            data=data_dir,
            courses=course_dirs,
            dis=do_import_static))
        try:
            mstore = modulestore('direct')
        except KeyError:
            self.stdout.write('Unable to load direct modulestore, trying '
                              'default\n')
            mstore = modulestore('default')

        _, course_items = import_from_xml(
            mstore, data_dir, course_dirs, load_error_modules=False,
            static_content_store=contentstore(), verbose=True,
            do_import_static=do_import_static,
            create_new_course=True,
        )

        for course in course_items:
            course_id = course.id
            if not are_permissions_roles_seeded(course_id):
                self.stdout.write('Seeding forum roles for course {0}\n'.format(course_id))
                seed_permissions_roles(course_id)

########NEW FILE########
__FILENAME__ = migrate_to_split
"""
Django management command to migrate a course from the old Mongo modulestore
to the new split-Mongo modulestore.
"""
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


def user_from_str(identifier):
    """
    Return a user identified by the given string. The string could be an email
    address, or a stringified integer corresponding to the ID of the user in
    the database. If no user could be found, a User.DoesNotExist exception
    will be raised.
    """
    try:
        user_id = int(identifier)
    except ValueError:
        return User.objects.get(email=identifier)
    else:
        return User.objects.get(id=user_id)


class Command(BaseCommand):
    "Migrate a course from old-Mongo to split-Mongo"

    help = "Migrate a course from old-Mongo to split-Mongo"
    args = "course_key email <new org> <new offering>"

    def parse_args(self, *args):
        """
        Return a 4-tuple of (course_key, user, org, offering).
        If the user didn't specify an org & offering, those will be None.
        """
        if len(args) < 2:
            raise CommandError(
                "migrate_to_split requires at least two arguments: "
                "a course_key and a user identifier (email or ID)"
            )

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        try:
            user = user_from_str(args[1])
        except User.DoesNotExist:
            raise CommandError("No user found identified by {}".format(args[1]))

        try:
            org = args[2]
            offering = args[3]
        except IndexError:
            org = offering = None

        return course_key, user, org, offering

    def handle(self, *args, **options):
        course_key, user, org, offering = self.parse_args(*args)

        migrator = SplitMigrator(
            draft_modulestore=modulestore('default'),
            direct_modulestore=modulestore('direct'),
            split_modulestore=modulestore('split'),
            loc_mapper=loc_mapper(),
        )

        migrator.migrate_mongo_course(course_key, user, org, offering)

########NEW FILE########
__FILENAME__ = populate_creators
"""
Script for granting existing course instructors course creator privileges.

This script is only intended to be run once on a given environment.
"""
from course_creators.views import add_user_with_status_granted, add_user_with_status_unrequested
from django.core.management.base import BaseCommand

from django.contrib.auth.models import User
from django.db.utils import IntegrityError
from student.roles import CourseInstructorRole, CourseStaffRole

#------------ to run: ./manage.py cms populate_creators --settings=dev
class Command(BaseCommand):
    """
    Script for granting existing course instructors course creator privileges.
    """
    help = 'Grants all users with INSTRUCTOR role permission to create courses'

    def handle(self, *args, **options):
        """
        The logic of the command.
        """
        username = 'populate_creators_command'
        email = 'grant+creator+access@edx.org'
        try:
            admin = User.objects.create_user(username, email, 'foo')
            admin.is_staff = True
            admin.save()
        except IntegrityError:
            # If the script did not complete the last time it was run,
            # the admin user will already exist.
            admin = User.objects.get(username=username, email=email)

        for user in get_users_with_role(CourseInstructorRole.ROLE):
            add_user_with_status_granted(admin, user)

        # Some users will be both staff and instructors. Those folks have been
        # added with status granted above, and add_user_with_status_unrequested
        # will not try to add them again if they already exist in the course creator database.
        for user in get_users_with_role(CourseStaffRole.ROLE):
            add_user_with_status_unrequested(user)

        # There could be users who are not in either staff or instructor (they've
        # never actually done anything in Studio). I plan to add those as unrequested
        # when they first go to their dashboard.

        admin.delete()


#=============================================================================================================
# Because these are expensive and far-reaching, I moved them here
def get_users_with_role(role_prefix):
    """
    An expensive operation which finds all users in the db with the given role prefix
    """
    return User.objects.filter(groups__name__startswith=role_prefix)

########NEW FILE########
__FILENAME__ = prompt
import sys


def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is one of "yes" or "no".
    """
    valid = {"yes": True,   "y": True,  "ye": True,
             "no": False,     "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        sys.stdout.write(question + prompt)
        choice = raw_input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")

########NEW FILE########
__FILENAME__ = restore_asset_from_trashcan
from django.core.management.base import BaseCommand, CommandError
from xmodule.contentstore.utils import restore_asset_from_trashcan


class Command(BaseCommand):
    help = '''Restore a deleted asset from the trashcan back to it's original course'''

    def handle(self, *args, **options):
        if len(args) != 1 and len(args) != 0:
            raise CommandError("restore_asset_from_trashcan requires one argument: <location>")

        restore_asset_from_trashcan(args[0])


########NEW FILE########
__FILENAME__ = rollback_split_course
"""
Django management command to rollback a migration to split. The way to do this
is to delete the course from the split mongo datastore.
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.locator import CourseLocator


class Command(BaseCommand):
    "Rollback a course that was migrated to the split Mongo datastore"

    help = "Rollback a course that was migrated to the split Mongo datastore"
    args = "org offering"

    def handle(self, *args, **options):
        if len(args) < 2:
            raise CommandError(
                "rollback_split_course requires 2 arguments (org offering)"
            )

        try:
            locator = CourseLocator(org=args[0], offering=args[1])
        except ValueError:
            raise CommandError("Invalid org or offering string {}, {}".format(*args))

        location = loc_mapper().translate_locator_to_location(locator, get_course=True)
        if not location:
            raise CommandError(
                "This course does not exist in the old Mongo store. "
                "This command is designed to rollback a course, not delete "
                "it entirely."
            )
        old_mongo_course = modulestore('direct').get_item(location)
        if not old_mongo_course:
            raise CommandError(
                "This course does not exist in the old Mongo store. "
                "This command is designed to rollback a course, not delete "
                "it entirely."
            )

        try:
            modulestore('split').delete_course(locator)
        except ItemNotFoundError:
            raise CommandError("No course found with locator {}".format(locator))

        print(
            'Course rolled back successfully. To delete this course entirely, '
            'call the "delete_course" management command.'
        )

########NEW FILE########
__FILENAME__ = test_course_id_clash
import sys
from StringIO import StringIO
from django.test import TestCase
from django.core.management import call_command
from xmodule.modulestore.tests.factories import CourseFactory

class ClashIdTestCase(TestCase):
    """
    Test for course_id_clash.
    """
    def test_course_clash(self):
        """
        Test for course_id_clash.
        """
        expected = []
        # clashing courses
        course = CourseFactory.create(org="test", course="courseid", display_name="run1")
        expected.append(course.id)
        course = CourseFactory.create(org="TEST", course="courseid", display_name="RUN12")
        expected.append(course.id)
        course = CourseFactory.create(org="test", course="CourseId", display_name="aRUN123")
        expected.append(course.id)
        # not clashing courses
        not_expected = []
        course = CourseFactory.create(org="test", course="course2", display_name="run1")
        not_expected.append(course.id)
        course = CourseFactory.create(org="test1", course="courseid", display_name="run1")
        not_expected.append(course.id)
        course = CourseFactory.create(org="test", course="courseid0", display_name="run1")
        not_expected.append(course.id)

        old_stdout = sys.stdout
        sys.stdout = mystdout = StringIO()
        call_command('course_id_clash', stdout=mystdout)
        sys.stdout = old_stdout
        result = mystdout.getvalue()
        for courseid in expected:
            self.assertIn(courseid.to_deprecated_string(), result)
        for courseid in not_expected:
            self.assertNotIn(courseid.to_deprecated_string(), result)

########NEW FILE########
__FILENAME__ = test_export_convert_format
"""
Test for export_convert_format.
"""
from unittest import TestCase
from django.core.management import call_command, CommandError
from tempfile import mkdtemp
import shutil
from path import path
from contentstore.management.commands.export_convert_format import Command, extract_source
from xmodule.tests.helpers import directories_equal


class ConvertExportFormat(TestCase):
    """
    Tests converting between export formats.
    """
    def setUp(self):
        """ Common setup. """
        self.temp_dir = mkdtemp()
        self.data_dir = path(__file__).realpath().parent / 'data'
        self.version0 = self.data_dir / "Version0_drafts.tar.gz"
        self.version1 = self.data_dir / "Version1_drafts.tar.gz"

        self.command = Command()

    def tearDown(self):
        """ Common cleanup. """
        shutil.rmtree(self.temp_dir)

    def test_no_args(self):
        """ Test error condition of no arguments. """
        errstring = "export requires two arguments"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle()

    def test_version1_archive(self):
        """
        Smoke test for creating a version 1 archive from a version 0.
        """
        call_command('export_convert_format', self.version0, self.temp_dir)
        output = path(self.temp_dir) / 'Version0_drafts_version_1.tar.gz'
        self.assertTrue(self._verify_archive_equality(output, self.version1))

    def test_version0_archive(self):
        """
        Smoke test for creating a version 0 archive from a version 1.
        """
        call_command('export_convert_format', self.version1, self.temp_dir)
        output = path(self.temp_dir) / 'Version1_drafts_version_0.tar.gz'
        self.assertTrue(self._verify_archive_equality(output, self.version0))

    def _verify_archive_equality(self, file1, file2):
        """
        Helper function for determining if 2 archives are equal.
        """
        temp_dir_1 = mkdtemp()
        temp_dir_2 = mkdtemp()
        try:
            extract_source(file1, temp_dir_1)
            extract_source(file2, temp_dir_2)
            return directories_equal(temp_dir_1, temp_dir_2)

        finally:
            shutil.rmtree(temp_dir_1)
            shutil.rmtree(temp_dir_2)

########NEW FILE########
__FILENAME__ = test_git_export
"""
Unittests for exporting to git via management command.
"""

import copy
import os
import shutil
import StringIO
import subprocess
import unittest
from uuid import uuid4

from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError
from django.test.utils import override_settings

from contentstore.tests.utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils
from contentstore.git_export_utils import GitExportError
from xmodule.modulestore.locations import SlashSeparatedCourseKey

FEATURES_WITH_EXPORT_GIT = settings.FEATURES.copy()
FEATURES_WITH_EXPORT_GIT['ENABLE_EXPORT_GIT'] = True
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
@override_settings(FEATURES=FEATURES_WITH_EXPORT_GIT)
class TestGitExport(CourseTestCase):
    """
    Excercise the git_export django management command with various inputs.
    """

    def setUp(self):
        """
        Create/reinitialize bare repo and folders needed
        """
        super(TestGitExport, self).setUp()

        if not os.path.isdir(git_export_utils.GIT_REPO_EXPORT_DIR):
            os.mkdir(git_export_utils.GIT_REPO_EXPORT_DIR)
            self.addCleanup(shutil.rmtree, git_export_utils.GIT_REPO_EXPORT_DIR)

        self.bare_repo_dir = '{0}/data/test_bare.git'.format(
            os.path.abspath(settings.TEST_ROOT))
        if not os.path.isdir(self.bare_repo_dir):
            os.mkdir(self.bare_repo_dir)
            self.addCleanup(shutil.rmtree, self.bare_repo_dir)
        subprocess.check_output(['git', '--bare', 'init'],
                                cwd=self.bare_repo_dir)

    def test_command(self):
        """
        Test that the command interface works. Ignore stderr for clean
        test output.
        """
        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, 'This script requires.*'):
                call_command('git_export', 'blah', 'blah', 'blah',
                             stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)

        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, 'This script requires.*'):
                call_command('git_export', stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)

        # Send bad url to get course not exported
        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, GitExportError.URL_BAD):
                call_command('git_export', 'foo/bar/baz', 'silly',
                             stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)
        # Send bad course_id to get course not exported
        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, GitExportError.BAD_COURSE):
                call_command('git_export', 'foo/bar:baz', 'silly',
                             stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)

    def test_bad_git_url(self):
        """
        Test several bad URLs for validation
        """
        course_key = SlashSeparatedCourseKey('org', 'course', 'run')
        with self.assertRaisesRegexp(GitExportError, str(GitExportError.URL_BAD)):
            git_export_utils.export_to_git(course_key, 'Sillyness')

        with self.assertRaisesRegexp(GitExportError, str(GitExportError.URL_BAD)):
            git_export_utils.export_to_git(course_key, 'example.com:edx/notreal')

        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.URL_NO_AUTH)):
            git_export_utils.export_to_git(course_key, 'http://blah')

    def test_bad_git_repos(self):
        """
        Test invalid git repos
        """
        test_repo_path = '{}/test_repo'.format(git_export_utils.GIT_REPO_EXPORT_DIR)
        self.assertFalse(os.path.isdir(test_repo_path))
        course_key = SlashSeparatedCourseKey('foo', 'blah', '100-')
        # Test bad clones
        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.CANNOT_PULL)):
            git_export_utils.export_to_git(
                course_key,
                'https://user:blah@example.com/test_repo.git')
        self.assertFalse(os.path.isdir(test_repo_path))

        # Setup good repo with bad course to test xml export
        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.XML_EXPORT_FAIL)):
            git_export_utils.export_to_git(
                course_key,
                'file://{0}'.format(self.bare_repo_dir))

        # Test bad git remote after successful clone
        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.CANNOT_PULL)):
            git_export_utils.export_to_git(
                course_key,
                'https://user:blah@example.com/r.git')

    @unittest.skipIf(os.environ.get('GIT_CONFIG') or
                     os.environ.get('GIT_AUTHOR_EMAIL') or
                     os.environ.get('GIT_AUTHOR_NAME') or
                     os.environ.get('GIT_COMMITTER_EMAIL') or
                     os.environ.get('GIT_COMMITTER_NAME'),
                     'Global git override set')
    def test_git_ident(self):
        """
        Test valid course with and without user specified.

        Test skipped if git global config override environment variable GIT_CONFIG
        is set.
        """
        git_export_utils.export_to_git(
            self.course.id,
            'file://{0}'.format(self.bare_repo_dir),
            'enigma'
        )
        expect_string = '{0}|{1}\n'.format(
            git_export_utils.GIT_EXPORT_DEFAULT_IDENT['name'],
            git_export_utils.GIT_EXPORT_DEFAULT_IDENT['email']
        )
        cwd = os.path.abspath(git_export_utils.GIT_REPO_EXPORT_DIR / 'test_bare')
        git_log = subprocess.check_output(['git', 'log', '-1',
                                           '--format=%an|%ae'], cwd=cwd)
        self.assertEqual(expect_string, git_log)

        # Make changes to course so there is something to commit
        self.populate_course()
        git_export_utils.export_to_git(
            self.course.id,
            'file://{0}'.format(self.bare_repo_dir),
            self.user.username
        )
        expect_string = '{0}|{1}\n'.format(
            self.user.username,
            self.user.email,
        )
        git_log = subprocess.check_output(
            ['git', 'log', '-1', '--format=%an|%ae'], cwd=cwd)
        self.assertEqual(expect_string, git_log)

    def test_no_change(self):
        """
        Test response if there are no changes
        """
        git_export_utils.export_to_git(
            self.course.id,
            'file://{0}'.format(self.bare_repo_dir)
        )

        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.CANNOT_COMMIT)):
            git_export_utils.export_to_git(
                self.course.id, 'file://{0}'.format(self.bare_repo_dir))

########NEW FILE########
__FILENAME__ = test_import
"""
Unittests for importing a course via management command
"""

import os
from path import path
import shutil
import tempfile

from django.core.management import call_command
from django.test.utils import override_settings

from contentstore.tests.modulestore_config import TEST_MODULESTORE
from django_comment_common.utils import are_permissions_roles_seeded
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.locations import SlashSeparatedCourseKey


@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestImport(ModuleStoreTestCase):
    """
    Unit tests for importing a course from command line
    """

    BASE_COURSE_KEY = SlashSeparatedCourseKey(u'edX', u'test_import_course', u'2013_Spring')
    DIFF_KEY = SlashSeparatedCourseKey(u'edX', u'test_import_course', u'2014_Spring')
    TRUNCATED_KEY = SlashSeparatedCourseKey(u'edX', u'test_import', u'2014_Spring')

    def create_course_xml(self, content_dir, course_id):
        directory = tempfile.mkdtemp(dir=content_dir)
        os.makedirs(os.path.join(directory, "course"))
        with open(os.path.join(directory, "course.xml"), "w+") as f:
            f.write('<course url_name="{0.run}" org="{0.org}" '
                    'course="{0.course}"/>'.format(course_id))

        with open(os.path.join(directory, "course", "{0.run}.xml".format(course_id)), "w+") as f:
            f.write('<course></course>')

        return directory

    def setUp(self):
        """
        Build course XML for importing
        """
        super(TestImport, self).setUp()
        self.content_dir = path(tempfile.mkdtemp())
        self.addCleanup(shutil.rmtree, self.content_dir)

        # Create good course xml
        self.good_dir = self.create_course_xml(self.content_dir, self.BASE_COURSE_KEY)

        # Create run changed course xml
        self.dupe_dir = self.create_course_xml(self.content_dir, self.DIFF_KEY)

        # Create course XML where TRUNCATED_COURSE.org == BASE_COURSE_ID.org
        # and BASE_COURSE_ID.startswith(TRUNCATED_COURSE.course)
        self.course_dir = self.create_course_xml(self.content_dir, self.TRUNCATED_KEY)

    def test_forum_seed(self):
        """
        Tests that forum roles were created with import.
        """
        self.assertFalse(are_permissions_roles_seeded(self.BASE_COURSE_KEY))
        call_command('import', self.content_dir, self.good_dir)
        self.assertTrue(are_permissions_roles_seeded(self.BASE_COURSE_KEY))

    def test_duplicate_with_url(self):
        """
        Check to make sure an import doesn't import courses that have the
        same org and course, but they have different runs in order to
        prevent modulestore "findone" exceptions on deletion
        """
        # Load up base course and verify it is available
        call_command('import', self.content_dir, self.good_dir)
        store = modulestore()
        self.assertIsNotNone(store.get_course(self.BASE_COURSE_KEY))

        # Now load up duped course and verify it doesn't load
        call_command('import', self.content_dir, self.dupe_dir)
        self.assertIsNone(store.get_course(self.DIFF_KEY))

    def test_truncated_course_with_url(self):
        """
        Check to make sure an import only blocks true duplicates: new
        courses with similar but not unique org/course combinations aren't
        blocked if the original course's course starts with the new course's
        org/course combinations (i.e. EDx/0.00x/Spring -> EDx/0.00/Spring)
        """
        # Load up base course and verify it is available
        call_command('import', self.content_dir, self.good_dir)
        store = modulestore()
        self.assertIsNotNone(store.get_course(self.BASE_COURSE_KEY))

        # Now load up the course with a similar course_id and verify it loads
        call_command('import', self.content_dir, self.course_dir)
        self.assertIsNotNone(store.get_course(self.TRUNCATED_KEY))

########NEW FILE########
__FILENAME__ = test_migrate_to_split
"""
Unittests for migrating a course to split mongo
"""
import unittest

from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.migrate_to_split import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, clear_existing_modulestores
from xmodule.modulestore.locator import CourseLocator
# pylint: disable=E1101


@unittest.skip("Not fixing split mongo until we land this long branch")
class TestArgParsing(unittest.TestCase):
    """
    Tests for parsing arguments for the `migrate_to_split` management command
    """
    def setUp(self):
        self.command = Command()

    def test_no_args(self):
        errstring = "migrate_to_split requires at least two arguments"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle()

    def test_invalid_location(self):
        errstring = "Invalid location string"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("foo", "bar")

    def test_nonexistant_user_id(self):
        errstring = "No user found identified by 99"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("i4x://org/course/category/name", "99")

    def test_nonexistant_user_email(self):
        errstring = "No user found identified by fake@example.com"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("i4x://org/course/category/name", "fake@example.com")


@unittest.skip("Not fixing split mongo until we land this long branch")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestMigrateToSplit(ModuleStoreTestCase):
    """
    Unit tests for migrating a course from old mongo to split mongo
    """

    def setUp(self):
        super(TestMigrateToSplit, self).setUp()
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'
        self.user = User.objects.create_user(uname, email, password)
        self.course = CourseFactory()
        self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
        self.addCleanup(clear_existing_modulestores)

    def test_user_email(self):
        call_command(
            "migrate_to_split",
            str(self.course.location),
            str(self.user.email),
        )
        course_from_split = modulestore('split').get_course(self.course.id)
        self.assertIsNotNone(course_from_split)

    def test_user_id(self):
        call_command(
            "migrate_to_split",
            str(self.course.location),
            str(self.user.id),
        )
        course_from_split = modulestore('split').get_course(self.course.id)
        self.assertIsNotNone(course_from_split)

    def test_locator_string(self):
        call_command(
            "migrate_to_split",
            str(self.course.location),
            str(self.user.id),
            "org.dept+name.run",
        )
        locator = CourseLocator(org="org.dept", offering="name.run", branch="published")
        course_from_split = modulestore('split').get_course(locator)
        self.assertIsNotNone(course_from_split)

########NEW FILE########
__FILENAME__ = test_rollback_split_course
"""
Unittests for deleting a split mongo course
"""
import unittest
from StringIO import StringIO
from mock import patch

from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.rollback_split_course import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.persistent_factories import PersistentCourseFactory
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.split_migrator import SplitMigrator
# pylint: disable=E1101


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
class TestArgParsing(unittest.TestCase):
    """
    Tests for parsing arguments for the `rollback_split_course` management command
    """
    def setUp(self):
        self.command = Command()

    def test_no_args(self):
        errstring = "rollback_split_course requires at least one argument"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle()

    def test_invalid_locator(self):
        errstring = "Invalid locator string !?!"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("!?!")


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
    """
    Unit tests for rolling back a split-mongo course from command line,
    where the course doesn't exist in the old mongo store
    """

    def setUp(self):
        super(TestRollbackSplitCourseNoOldMongo, self).setUp()
        self.course = PersistentCourseFactory()

    def test_no_old_course(self):
        locator = self.course.location
        errstring = "course does not exist in the old Mongo store"
        with self.assertRaisesRegexp(CommandError, errstring):
            Command().handle(str(locator))


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
    """
    Unit tests for rolling back a split-mongo course from command line,
    where the course doesn't exist in the split mongo store
    """

    def setUp(self):
        super(TestRollbackSplitCourseNoSplitMongo, self).setUp()
        self.old_course = CourseFactory()

    def test_nonexistent_locator(self):
        locator = loc_mapper().translate_location(self.old_course.location)
        errstring = "No course found with locator"
        with self.assertRaisesRegexp(CommandError, errstring):
            Command().handle(str(locator))


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourse(ModuleStoreTestCase):
    """
    Unit tests for rolling back a split-mongo course from command line
    """
    def setUp(self):
        super(TestRollbackSplitCourse, self).setUp()
        self.old_course = CourseFactory()
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'
        self.user = User.objects.create_user(uname, email, password)

        # migrate old course to split
        migrator = SplitMigrator(
            draft_modulestore=modulestore('default'),
            direct_modulestore=modulestore('direct'),
            split_modulestore=modulestore('split'),
            loc_mapper=loc_mapper(),
        )
        migrator.migrate_mongo_course(self.old_course.location, self.user)
        self.course = modulestore('split').get_course(self.old_course.id)

    @patch("sys.stdout", new_callable=StringIO)
    def test_happy_path(self, mock_stdout):
        course_id = self.course.id
        call_command(
            "rollback_split_course",
            str(course_id),
        )
        with self.assertRaises(ItemNotFoundError):
            modulestore('split').get_course(course_id)

        self.assertIn("Course rolled back successfully", mock_stdout.getvalue())


########NEW FILE########
__FILENAME__ = xlint
"""
Verify the structure of courseware as to it's suitability for import
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_importer import perform_xlint


class Command(BaseCommand):
    """Verify the structure of courseware as to it's suitability for import"""
    help = "Verify the structure of courseware as to it's suitability for import"

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) == 0:
            raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")

        data_dir = args[0]
        if len(args) > 1:
            course_dirs = args[1:]
        else:
            course_dirs = None
        print("Importing.  Data_dir={data}, course_dirs={courses}".format(
            data=data_dir,
            courses=course_dirs))
        perform_xlint(data_dir, course_dirs, load_error_modules=False)

########NEW FILE########
__FILENAME__ = modulestore_config
"""
Define test configuration for modulestores.
"""

from xmodule.modulestore.tests.django_utils import studio_store_config
from django.conf import settings

TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")

########NEW FILE########
__FILENAME__ = tests
"""
This test file will test registration, login, activation, and session activity timeouts
"""
import time
import mock
import unittest

from django.test.utils import override_settings
from django.core.cache import cache
from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from contentstore.tests.utils import parse_json, user, registration, AjaxEnabledTestClient
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.test_course_settings import CourseTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
import datetime
from pytz import UTC

from freezegun import freeze_time


@override_settings(MODULESTORE=TEST_MODULESTORE)
class ContentStoreTestCase(ModuleStoreTestCase):
    def _login(self, email, password):
        """
        Login.  View should always return 200.  The success/fail is in the
        returned json
        """
        resp = self.client.post(
            reverse('login_post'),
            {'email': email, 'password': password}
        )
        self.assertEqual(resp.status_code, 200)
        return resp

    def login(self, email, password):
        """Login, check that it worked."""
        resp = self._login(email, password)
        data = parse_json(resp)
        self.assertTrue(data['success'])
        return resp

    def _create_account(self, username, email, password):
        """Try to create an account.  No error checking"""
        resp = self.client.post('/create_account', {
            'username': username,
            'email': email,
            'password': password,
            'location': 'home',
            'language': 'Franglish',
            'name': 'Fred Weasley',
            'terms_of_service': 'true',
            'honor_code': 'true',
        })
        return resp

    def create_account(self, username, email, password):
        """Create the account and check that it worked"""
        resp = self._create_account(username, email, password)
        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertEqual(data['success'], True)

        # Check both that the user is created, and inactive
        self.assertFalse(user(email).is_active)

        return resp

    def _activate_user(self, email):
        """Look up the activation key for the user, then hit the activate view.
        No error checking"""
        activation_key = registration(email).activation_key

        # and now we try to activate
        resp = self.client.get(reverse('activate', kwargs={'key': activation_key}))
        return resp

    def activate_user(self, email):
        resp = self._activate_user(email)
        self.assertEqual(resp.status_code, 200)
        # Now make sure that the user is now actually activated
        self.assertTrue(user(email).is_active)


class AuthTestCase(ContentStoreTestCase):
    """Check that various permissions-related things work"""

    def setUp(self):
        self.email = 'a@b.com'
        self.pw = 'xyz'
        self.username = 'testuser'
        self.client = AjaxEnabledTestClient()
        # clear the cache so ratelimiting won't affect these tests
        cache.clear()

    def check_page_get(self, url, expected):
        resp = self.client.get_html(url)
        self.assertEqual(resp.status_code, expected)
        return resp

    def test_public_pages_load(self):
        """Make sure pages that don't require login load without error."""
        pages = (
            reverse('login'),
            reverse('signup'),
        )
        for page in pages:
            print("Checking '{0}'".format(page))
            self.check_page_get(page, 200)

    def test_create_account_errors(self):
        # No post data -- should fail
        resp = self.client.post('/create_account', {})
        self.assertEqual(resp.status_code, 400)
        data = parse_json(resp)
        self.assertEqual(data['success'], False)

    def test_create_account(self):
        self.create_account(self.username, self.email, self.pw)
        self.activate_user(self.email)

    def test_create_account_username_already_exists(self):
        User.objects.create_user(self.username, self.email, self.pw)
        resp = self._create_account(self.username, "abc@def.com", "password")
        # we have a constraint on unique usernames, so this should fail
        self.assertEqual(resp.status_code, 400)

    def test_create_account_pw_already_exists(self):
        User.objects.create_user(self.username, self.email, self.pw)
        resp = self._create_account("abcdef", "abc@def.com", self.pw)
        # we can have two users with the same password, so this should succeed
        self.assertEqual(resp.status_code, 200)

    @unittest.skipUnless(settings.SOUTH_TESTS_MIGRATE, "South migrations required")
    def test_create_account_email_already_exists(self):
        User.objects.create_user(self.username, self.email, self.pw)
        resp = self._create_account("abcdef", self.email, "password")
        # This is tricky. Django's user model doesn't have a constraint on
        # unique email addresses, but we *add* that constraint during the
        # migration process:
        # see common/djangoapps/student/migrations/0004_add_email_index.py
        #
        # The behavior we *want* is for this account creation request
        # to fail, due to this uniqueness constraint, but the request will
        # succeed if the migrations have not run.
        self.assertEqual(resp.status_code, 400)

    def test_login(self):
        self.create_account(self.username, self.email, self.pw)

        # Not activated yet.  Login should fail.
        resp = self._login(self.email, self.pw)
        data = parse_json(resp)
        self.assertFalse(data['success'])

        self.activate_user(self.email)

        # Now login should work
        self.login(self.email, self.pw)

    def test_login_ratelimited(self):
        # try logging in 30 times, the default limit in the number of failed
        # login attempts in one 5 minute period before the rate gets limited
        for i in xrange(30):
            resp = self._login(self.email, 'wrong_password{0}'.format(i))
            self.assertEqual(resp.status_code, 200)
        resp = self._login(self.email, 'wrong_password')
        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertFalse(data['success'])
        self.assertIn('Too many failed login attempts.', data['value'])

    @override_settings(MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED=3)
    @override_settings(MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS=2)
    def test_excessive_login_failures(self):
        # try logging in 3 times, the account should get locked for 3 seconds
        # note we want to keep the lockout time short, so we don't slow down the tests

        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MAX_FAILED_LOGIN_ATTEMPTS': True}):
            self.create_account(self.username, self.email, self.pw)
            self.activate_user(self.email)

            for i in xrange(3):
                resp = self._login(self.email, 'wrong_password{0}'.format(i))
                self.assertEqual(resp.status_code, 200)
                data = parse_json(resp)
                self.assertFalse(data['success'])
                self.assertIn(
                    'Email or password is incorrect.',
                    data['value']
                )

            # now the account should be locked

            resp = self._login(self.email, 'wrong_password')
            self.assertEqual(resp.status_code, 200)
            data = parse_json(resp)
            self.assertFalse(data['success'])
            self.assertIn(
                'This account has been temporarily locked due to excessive login failures. Try again later.',
                data['value']
            )

            with freeze_time('2100-01-01'):
                self.login(self.email, self.pw)

            # make sure the failed attempt counter gets reset on successful login
            resp = self._login(self.email, 'wrong_password')
            self.assertEqual(resp.status_code, 200)
            data = parse_json(resp)
            self.assertFalse(data['success'])

            # account should not be locked out after just one attempt
            self.login(self.email, self.pw)

            # do one more login when there is no bad login counter row at all in the database to
            # test the "ObjectNotFound" case
            self.login(self.email, self.pw)

    def test_login_link_on_activation_age(self):
        self.create_account(self.username, self.email, self.pw)
        # we want to test the rendering of the activation page when the user isn't logged in
        self.client.logout()
        resp = self._activate_user(self.email)
        self.assertEqual(resp.status_code, 200)

        # check the the HTML has links to the right login page. Note that this is merely a content
        # check and thus could be fragile should the wording change on this page
        expected = 'You can now <a href="' + reverse('login') + '">login</a>.'
        self.assertIn(expected, resp.content)

    def test_private_pages_auth(self):
        """Make sure pages that do require login work."""
        auth_pages = (
            '/course/',
        )

        # These are pages that should just load when the user is logged in
        # (no data needed)
        simple_auth_pages = (
            '/course/',
        )

        # need an activated user
        self.test_create_account()

        # Create a new session
        self.client = AjaxEnabledTestClient()

        # Not logged in.  Should redirect to login.
        print('Not logged in')
        for page in auth_pages:
            print("Checking '{0}'".format(page))
            self.check_page_get(page, expected=302)

        # Logged in should work.
        self.login(self.email, self.pw)

        print('Logged in')
        for page in simple_auth_pages:
            print("Checking '{0}'".format(page))
            self.check_page_get(page, expected=200)

    def test_index_auth(self):

        # not logged in.  Should return a redirect.
        resp = self.client.get_html('/course/')
        self.assertEqual(resp.status_code, 302)

        # Logged in should work.

    @override_settings(SESSION_INACTIVITY_TIMEOUT_IN_SECONDS=1)
    def test_inactive_session_timeout(self):
        """
        Verify that an inactive session times out and redirects to the
        login page
        """
        self.create_account(self.username, self.email, self.pw)
        self.activate_user(self.email)

        self.login(self.email, self.pw)

        # make sure we can access courseware immediately
        course_url = '/course/'
        resp = self.client.get_html(course_url)
        self.assertEquals(resp.status_code, 200)

        # then wait a bit and see if we get timed out
        time.sleep(2)

        resp = self.client.get_html(course_url)

        # re-request, and we should get a redirect to login page
        self.assertRedirects(resp, settings.LOGIN_REDIRECT_URL + '?next=/course/')


class ForumTestCase(CourseTestCase):
    def setUp(self):
        """ Creates the test course. """
        super(ForumTestCase, self).setUp()
        self.course = CourseFactory.create(org='testX', number='727', display_name='Forum Course')

    def test_blackouts(self):
        now = datetime.datetime.now(UTC)
        times1 = [
            (now - datetime.timedelta(days=14), now - datetime.timedelta(days=11)),
            (now + datetime.timedelta(days=24), now + datetime.timedelta(days=30))
        ]
        self.course.discussion_blackouts = [(t.isoformat(), t2.isoformat()) for t, t2 in times1]
        self.assertTrue(self.course.forum_posts_allowed)
        times2 = [
            (now - datetime.timedelta(days=14), now + datetime.timedelta(days=2)),
            (now + datetime.timedelta(days=24), now + datetime.timedelta(days=30))
        ]
        self.course.discussion_blackouts = [(t.isoformat(), t2.isoformat()) for t, t2 in times2]
        self.assertFalse(self.course.forum_posts_allowed)

########NEW FILE########
__FILENAME__ = test_contentstore
# -*- coding: utf-8 -*-
# pylint: disable=E1101

import copy
import mock
import re
import shutil

from datetime import timedelta
from fs.osfs import OSFS
from json import loads
from path import path
from pymongo import MongoClient
from tempdir import mkdtemp_clean
from textwrap import dedent
from uuid import uuid4

from django.conf import settings
from django.contrib.auth.models import User
from django.test import TestCase
from django.test.utils import override_settings

from contentstore.utils import get_modulestore
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import parse_json, AjaxEnabledTestClient
from contentstore.views.component import ADVANCED_COMPONENT_TYPES

from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan
from xmodule.exceptions import NotFoundError, InvalidVersionError
from xmodule.modulestore import mongo
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.keys import UsageKey
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.store_utilities import clone_course, delete_course
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint

from xmodule.capa_module import CapaDescriptor
from xmodule.course_module import CourseDescriptor
from xmodule.seq_module import SequenceDescriptor

from contentstore.utils import delete_course_and_groups, reverse_url, reverse_course_url
from django_comment_common.utils import are_permissions_roles_seeded

from student import auth
from student.models import CourseEnrollment
from student.roles import CourseCreatorRole, CourseInstructorRole
from opaque_keys import InvalidKeyError


TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


class MongoCollectionFindWrapper(object):
    def __init__(self, original):
        self.original = original
        self.counter = 0

    def find(self, query, *args, **kwargs):
        self.counter = self.counter + 1
        return self.original(query, *args, **kwargs)


def get_url(handler_name, key_value, key_name='usage_key_string', kwargs=None):
    # Helper function for getting HTML for a page in Studio and
    # checking that it does not error.
    return reverse_url(handler_name, key_name, key_value, kwargs)


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
    """
    Tests that rely on the toy courses.
    TODO: refactor using CourseFactory so they do not.
    """
    def setUp(self):

        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True

        # Save the data that we've just changed to the db.
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def check_components_on_page(self, component_types, expected_types):
        """
        Ensure that the right types end up on the page.

        component_types is the list of advanced components.

        expected_types is the list of elements that should appear on the page.

        expected_types and component_types should be similar, but not
        exactly the same -- for example, 'video' in
        component_types should cause 'Video' to be present.
        """
        store = modulestore('direct')
        _, course_items = import_from_xml(store, 'common/test/data/', ['simple'])
        course = course_items[0]
        course.advanced_modules = component_types
        store.update_item(course, self.user.id)

        # just pick one vertical
        descriptor = store.get_items(course.id, category='vertical',)
        resp = self.client.get_html(get_url('unit_handler', descriptor[0].location))
        self.assertEqual(resp.status_code, 200)
        _test_no_locations(self, resp)

        for expected in expected_types:
            self.assertIn(expected, resp.content)

    def test_advanced_components_in_edit_unit(self):
        # This could be made better, but for now let's just assert that we see the advanced modules mentioned in the page
        # response HTML
        self.check_components_on_page(
            ADVANCED_COMPONENT_TYPES,
            ['Word cloud', 'Annotation', 'Text Annotation', 'Video Annotation', 'Image Annotation',
             'Open Response Assessment', 'Peer Grading Interface', 'openassessment'],
        )

    def test_advanced_components_require_two_clicks(self):
        self.check_components_on_page(['word_cloud'], ['Word cloud'])

    def test_malformed_edit_unit_request(self):
        store = modulestore('direct')
        _, course_items = import_from_xml(store, 'common/test/data/', ['simple'])

        # just pick one vertical
        usage_key = course_items[0].id.make_usage_key('vertical', None)

        resp = self.client.get_html(get_url('unit_handler', usage_key))
        self.assertEqual(resp.status_code, 400)
        _test_no_locations(self, resp, status_code=400)

    def check_edit_unit(self, test_course_name):
        _, course_items = import_from_xml(modulestore('direct'), 'common/test/data/', [test_course_name])

        items = modulestore().get_items(course_items[0].id, category='vertical')
        self._check_verticals(items)

    def _lock_an_asset(self, content_store, course_id):
        """
        Lock an arbitrary asset in the course
        :param course_location:
        """
        course_assets, __ = content_store.get_all_content_for_course(course_id)
        self.assertGreater(len(course_assets), 0, "No assets to lock")
        asset_id = course_assets[0]['_id']
        asset_key = StaticContent.compute_location(course_id, asset_id['name'])
        content_store.set_attr(asset_key, 'locked', True)
        return asset_key

    def test_edit_unit_toy(self):
        self.check_edit_unit('toy')

    def _get_draft_counts(self, item):
        cnt = 1 if getattr(item, 'is_draft', False) else 0
        for child in item.get_children():
            cnt = cnt + self._get_draft_counts(child)

        return cnt

    def test_get_items(self):
        '''
        This verifies a bug we had where the None setting in get_items() meant 'wildcard'
        Unfortunately, None = published for the revision field, so get_items() would return
        both draft and non-draft copies.
        '''
        direct_store = modulestore('direct')
        draft_store = modulestore('draft')
        _, course_items = import_from_xml(direct_store, 'common/test/data/', ['simple'])
        course_key = course_items[0].id
        html_usage_key = course_key.make_usage_key('html', 'test_html')

        html_module_from_draft_store = draft_store.get_item(html_usage_key)
        draft_store.convert_to_draft(html_module_from_draft_store.location)

        # Query get_items() and find the html item. This should just return back a single item (not 2).

        direct_store_items = direct_store.get_items(course_key)
        html_items_from_direct_store = [item for item in direct_store_items if (item.location == html_usage_key)]
        self.assertEqual(len(html_items_from_direct_store), 1)
        self.assertFalse(getattr(html_items_from_direct_store[0], 'is_draft', False))

        # Fetch from the draft store. Note that even though we pass
        # None in the revision field, the draft store will replace that with 'draft'.
        draft_store_items = draft_store.get_items(course_key)
        html_items_from_draft_store = [item for item in draft_store_items if (item.location == html_usage_key)]
        self.assertEqual(len(html_items_from_draft_store), 1)
        self.assertTrue(getattr(html_items_from_draft_store[0], 'is_draft', False))


    def test_draft_metadata(self):
        '''
        This verifies a bug we had where inherited metadata was getting written to the
        module as 'own-metadata' when publishing. Also verifies the metadata inheritance is
        properly computed
        '''
        store = modulestore('direct')
        draft_store = modulestore('draft')
        import_from_xml(store, 'common/test/data/', ['simple'])

        course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
        html_usage_key = course_key.make_usage_key('html', 'test_html')
        course = draft_store.get_course(course_key)
        html_module = draft_store.get_item(html_usage_key)

        self.assertEqual(html_module.graceperiod, course.graceperiod)
        self.assertNotIn('graceperiod', own_metadata(html_module))

        draft_store.convert_to_draft(html_module.location)

        # refetch to check metadata
        html_module = draft_store.get_item(html_usage_key)

        self.assertEqual(html_module.graceperiod, course.graceperiod)
        self.assertNotIn('graceperiod', own_metadata(html_module))

        # publish module
        draft_store.publish(html_module.location, 0)

        # refetch to check metadata
        html_module = draft_store.get_item(html_usage_key)

        self.assertEqual(html_module.graceperiod, course.graceperiod)
        self.assertNotIn('graceperiod', own_metadata(html_module))

        # put back in draft and change metadata and see if it's now marked as 'own_metadata'
        draft_store.convert_to_draft(html_module.location)
        html_module = draft_store.get_item(html_usage_key)

        new_graceperiod = timedelta(hours=1)

        self.assertNotIn('graceperiod', own_metadata(html_module))
        html_module.graceperiod = new_graceperiod
        # Save the data that we've just changed to the underlying
        # MongoKeyValueStore before we update the mongo datastore.
        html_module.save()
        self.assertIn('graceperiod', own_metadata(html_module))
        self.assertEqual(html_module.graceperiod, new_graceperiod)

        draft_store.update_item(html_module, self.user.id)

        # read back to make sure it reads as 'own-metadata'
        html_module = draft_store.get_item(html_usage_key)

        self.assertIn('graceperiod', own_metadata(html_module))
        self.assertEqual(html_module.graceperiod, new_graceperiod)

        # republish
        draft_store.publish(html_module.location, 0)

        # and re-read and verify 'own-metadata'
        draft_store.convert_to_draft(html_module.location)
        html_module = draft_store.get_item(html_usage_key)

        self.assertIn('graceperiod', own_metadata(html_module))
        self.assertEqual(html_module.graceperiod, new_graceperiod)

    def test_get_depth_with_drafts(self):
        import_from_xml(modulestore('direct'), 'common/test/data/', ['simple'])

        course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
        course = modulestore('draft').get_course(course_key)

        # make sure no draft items have been returned
        num_drafts = self._get_draft_counts(course)
        self.assertEqual(num_drafts, 0)

        problem_usage_key = course_key.make_usage_key('problem', 'ps01-simple')
        problem = modulestore('draft').get_item(problem_usage_key)

        # put into draft
        modulestore('draft').convert_to_draft(problem.location)

        # make sure we can query that item and verify that it is a draft
        draft_problem = modulestore('draft').get_item(problem_usage_key)
        self.assertTrue(getattr(draft_problem, 'is_draft', False))

        # now requery with depth
        course = modulestore('draft').get_course(course_key)

        # make sure just one draft item have been returned
        num_drafts = self._get_draft_counts(course)
        self.assertEqual(num_drafts, 1)

    def test_no_static_link_rewrites_on_import(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course = course_items[0]

        handouts_usage_key = course.id.make_usage_key('course_info', 'handouts')
        handouts = module_store.get_item(handouts_usage_key)
        self.assertIn('/static/', handouts.data)

        handouts_usage_key = course.id.make_usage_key('html', 'toyhtml')
        handouts = module_store.get_item(handouts_usage_key)
        self.assertIn('/static/', handouts.data)

    @mock.patch('xmodule.course_module.requests.get')
    def test_import_textbook_as_content_element(self, mock_get):
        mock_get.return_value.text = dedent("""
            <?xml version="1.0"?><table_of_contents>
            <entry page="5" page_label="ii" name="Table of Contents"/>
            </table_of_contents>
        """).strip()

        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
        self.assertGreater(len(course.textbooks), 0)

    def test_import_polls(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_key = course_items[0].id

        items = module_store.get_items(course_key, category='poll_question')
        found = len(items) > 0

        self.assertTrue(found)
        # check that there's actually content in the 'question' field
        self.assertGreater(len(items[0].question), 0)

    def test_xlint_fails(self):
        err_cnt = perform_xlint('common/test/data', ['toy'])
        self.assertGreater(err_cnt, 0)

    @override_settings(COURSES_WITH_UNSAFE_CODE=['edX/toy/.*'])
    def test_module_preview_in_whitelist(self):
        """
        Tests the ajax callback to render an XModule
        """
        direct_store = modulestore('direct')
        _, course_items = import_from_xml(direct_store, 'common/test/data/', ['toy'])
        usage_key = course_items[0].id.make_usage_key('vertical', 'vertical_test')

        # also try a custom response which will trigger the 'is this course in whitelist' logic
        resp = self.client.get_json(
            get_url('xblock_view_handler', usage_key, kwargs={'view_name': 'container_preview'})
        )
        self.assertEqual(resp.status_code, 200)
        # TODO: uncomment when preview no longer has locations being returned.
        # _test_no_locations(self, resp)

        # These are the data-ids of the xblocks contained in the vertical.
        self.assertContains(resp, 'edX+toy+2012_Fall+video+sample_video')
        self.assertContains(resp, 'edX+toy+2012_Fall+video+separate_file_video')
        self.assertContains(resp, 'edX+toy+2012_Fall+video+video_with_end_time')
        self.assertContains(resp, 'edX+toy+2012_Fall+poll_question+T1_changemind_poll_foo_2')

    def test_delete(self):
        direct_store = modulestore('direct')
        course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')

        chapterloc = ItemFactory.create(parent_location=course.location, display_name="Chapter").location
        ItemFactory.create(parent_location=chapterloc, category='sequential', display_name="Sequential")

        sequential_key = course.id.make_usage_key('sequential', 'Sequential')
        sequential = direct_store.get_item(sequential_key)
        chapter_key = course.id.make_usage_key('chapter', 'Chapter')
        chapter = direct_store.get_item(chapter_key)

        # make sure the parent points to the child object which is to be deleted
        self.assertTrue(sequential.location in chapter.children)

        self.client.delete(get_url('xblock_handler', sequential_key), {'recurse': True, 'all_versions': True})

        found = False
        try:
            direct_store.get_item(sequential_key)
            found = True
        except ItemNotFoundError:
            pass

        self.assertFalse(found)

        chapter = direct_store.get_item(chapter_key)

        # make sure the parent no longer points to the child object which was deleted
        self.assertFalse(sequential.location in chapter.children)

    def test_about_overrides(self):
        '''
        This test case verifies that a course can use specialized override for about data, e.g. /about/Fall_2012/effort.html
        while there is a base definition in /about/effort.html
        '''
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_key = course_items[0].id
        effort = module_store.get_item(course_key.make_usage_key('about', 'effort'))
        self.assertEqual(effort.data, '6 hours')

        # this one should be in a non-override folder
        effort = module_store.get_item(course_key.make_usage_key('about', 'end_date'))
        self.assertEqual(effort.data, 'TBD')

    def test_asset_import(self):
        '''
        This test validates that an image asset is imported and a thumbnail was generated for a .gif
        '''
        content_store = contentstore()

        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)

        course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))

        self.assertIsNotNone(course)

        # make sure we have some assets in our contentstore
        all_assets, __ = content_store.get_all_content_for_course(course.id)
        self.assertGreater(len(all_assets), 0)

        # make sure we have some thumbnails in our contentstore
        content_store.get_all_content_thumbnails_for_course(course.id)

        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        #
        # self.assertGreater(len(all_thumbnails), 0)

        content = None
        try:
            location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
            content = content_store.find(location)
        except NotFoundError:
            pass

        self.assertIsNotNone(content)

        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        # self.assertIsNotNone(content.thumbnail_location)
        #
        # thumbnail = None
        # try:
        #    thumbnail = content_store.find(content.thumbnail_location)
        # except:
        #    pass
        #
        # self.assertIsNotNone(thumbnail)

    def test_asset_delete_and_restore(self):
        '''
        This test will exercise the soft delete/restore functionality of the assets
        '''
        content_store, trash_store, thumbnail_location, _location = self._delete_asset_in_course()
        asset_location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')

        # now try to find it in store, but they should not be there any longer
        content = content_store.find(asset_location, throw_on_not_found=False)
        self.assertIsNone(content)

        if thumbnail_location:
            thumbnail = content_store.find(thumbnail_location, throw_on_not_found=False)
            self.assertIsNone(thumbnail)

        # now try to find it and the thumbnail in trashcan - should be in there
        content = trash_store.find(asset_location, throw_on_not_found=False)
        self.assertIsNotNone(content)

        if thumbnail_location:
            thumbnail = trash_store.find(thumbnail_location, throw_on_not_found=False)
            self.assertIsNotNone(thumbnail)

        # let's restore the asset
        restore_asset_from_trashcan('/c4x/edX/toy/asset/sample_static.txt')

        # now try to find it in courseware store, and they should be back after restore
        content = content_store.find(asset_location, throw_on_not_found=False)
        self.assertIsNotNone(content)

        if thumbnail_location:
            thumbnail = content_store.find(thumbnail_location, throw_on_not_found=False)
            self.assertIsNotNone(thumbnail)

    def _delete_asset_in_course(self):
        """
        Helper method for:
          1) importing course from xml
          2) finding asset in course (verifying non-empty)
          3) computing thumbnail location of asset
          4) deleting the asset from the course
        """

        content_store = contentstore()
        trash_store = contentstore('trashcan')
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)

        # look up original (and thumbnail) in content store, should be there after import
        location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
        content = content_store.find(location, throw_on_not_found=False)
        thumbnail_location = content.thumbnail_location
        self.assertIsNotNone(content)

        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        # self.assertIsNotNone(thumbnail_location)

        # go through the website to do the delete, since the soft-delete logic is in the view
        course = course_items[0]
        url = reverse_course_url(
            'assets_handler',
            course.id,
            kwargs={'asset_key_string': course.id.make_asset_key('asset', 'sample_static.txt')}
        )
        resp = self.client.delete(url)
        self.assertEqual(resp.status_code, 204)

        return content_store, trash_store, thumbnail_location, location

    def test_course_info_updates_import_export(self):
        """
        Test that course info updates are imported and exported with all content fields ('data', 'items')
        """
        content_store = contentstore()
        module_store = modulestore('direct')
        data_dir = "common/test/data/"
        import_from_xml(module_store, data_dir, ['course_info_updates'],
                        static_content_store=content_store, verbose=True)

        course_id = SlashSeparatedCourseKey('edX', 'course_info_updates', '2014_T1')
        course = module_store.get_course(course_id)

        self.assertIsNotNone(course)

        course_updates = module_store.get_item(course_id.make_usage_key('course_info', 'updates'))

        self.assertIsNotNone(course_updates)

        # check that course which is imported has files 'updates.html' and 'updates.items.json'
        filesystem = OSFS(data_dir + 'course_info_updates/info')
        self.assertTrue(filesystem.exists('updates.html'))
        self.assertTrue(filesystem.exists('updates.items.json'))

        # verify that course info update module has same data content as in data file from which it is imported
        # check 'data' field content
        with filesystem.open('updates.html', 'r') as course_policy:
            on_disk = course_policy.read()
            self.assertEqual(course_updates.data, on_disk)

        # check 'items' field content
        with filesystem.open('updates.items.json', 'r') as course_policy:
            on_disk = loads(course_policy.read())
            self.assertEqual(course_updates.items, on_disk)

        # now export the course to a tempdir and test that it contains files 'updates.html' and 'updates.items.json'
        # with same content as in course 'info' directory
        root_dir = path(mkdtemp_clean())
        print 'Exporting to tempdir = {0}'.format(root_dir)
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export')

        # check that exported course has files 'updates.html' and 'updates.items.json'
        filesystem = OSFS(root_dir / 'test_export/info')
        self.assertTrue(filesystem.exists('updates.html'))
        self.assertTrue(filesystem.exists('updates.items.json'))

        # verify that exported course has same data content as in course_info_update module
        with filesystem.open('updates.html', 'r') as grading_policy:
            on_disk = grading_policy.read()
            self.assertEqual(on_disk, course_updates.data)

        with filesystem.open('updates.items.json', 'r') as grading_policy:
            on_disk = loads(grading_policy.read())
            self.assertEqual(on_disk, course_updates.items)

    def test_empty_trashcan(self):
        '''
        This test will exercise the emptying of the asset trashcan
        '''
        __, trash_store, __, _location = self._delete_asset_in_course()

        # make sure there's something in the trashcan
        course_id = SlashSeparatedCourseKey('edX', 'toy', '6.002_Spring_2012')
        all_assets, __ = trash_store.get_all_content_for_course(course_id)
        self.assertGreater(len(all_assets), 0)

        # make sure we have some thumbnails in our trashcan
        _all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_id)
        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        # self.assertGreater(len(all_thumbnails), 0)

        # empty the trashcan
        empty_asset_trashcan([course_id])

        # make sure trashcan is empty
        all_assets, count = trash_store.get_all_content_for_course(course_id)
        self.assertEqual(len(all_assets), 0)
        self.assertEqual(count, 0)

        all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_id)
        self.assertEqual(len(all_thumbnails), 0)

    def test_clone_course(self):

        course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
            'run': '2013_Spring',
        }

        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])

        source_course_id = course_items[0].id
        dest_course_id = _get_course_id(course_data)

        # get a vertical (and components in it) to put into 'draft'
        # this is to assert that draft content is also cloned over
        vertical = module_store.get_item(
            source_course_id.make_usage_key('vertical', 'vertical_test'),
            depth=1
        )

        draft_store.convert_to_draft(vertical.location)
        for child in vertical.get_children():
            draft_store.convert_to_draft(child.location)

        items = module_store.get_items(source_course_id, revision='draft')
        self.assertGreater(len(items), 0)

        _create_course(self, dest_course_id, course_data)

        content_store = contentstore()

        # now do the actual cloning
        clone_course(module_store, content_store, source_course_id, dest_course_id)

        # first assert that all draft content got cloned as well
        items = module_store.get_items(source_course_id, revision='draft')
        self.assertGreater(len(items), 0)
        clone_items = module_store.get_items(dest_course_id, revision='draft')
        self.assertGreater(len(clone_items), 0)
        self.assertEqual(len(items), len(clone_items))

        # now loop through all the units in the course and verify that the clone can render them, which
        # means the objects are at least present
        items = module_store.get_items(source_course_id, revision=None)
        self.assertGreater(len(items), 0)
        clone_items = module_store.get_items(dest_course_id, revision=None)
        self.assertGreater(len(clone_items), 0)

        for descriptor in items:
            source_item = module_store.get_item(descriptor.location)
            new_loc = descriptor.location.map_into_course(dest_course_id)
            print "Checking {0} should now also be at {1}".format(descriptor.location, new_loc)
            lookup_item = module_store.get_item(new_loc)

            if hasattr(source_item, 'data') and hasattr(lookup_item, 'data'):
                self.assertEqual(source_item.data, lookup_item.data)

            # also make sure that metadata was cloned over and filtered with own_metadata, i.e. inherited
            # values were not explicitly set
            self.assertEqual(own_metadata(source_item), own_metadata(lookup_item))

            # check that the children are as expected
            self.assertEqual(source_item.has_children, lookup_item.has_children)
            if source_item.has_children:
                expected_children = []
                for child_loc in source_item.children:
                    child_loc = child_loc.map_into_course(dest_course_id)
                    expected_children.append(child_loc)
                self.assertEqual(expected_children, lookup_item.children)

    def test_portable_link_rewrites_during_clone_course(self):
        course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
            'run': '2013_Spring'
        }

        module_store = modulestore('direct')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])

        source_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
        dest_course_id = _get_course_id(course_data)

        # let's force a non-portable link in the clone source
        # as a final check, make sure that any non-portable links are rewritten during cloning
        html_module = module_store.get_item(source_course_id.make_usage_key('html', 'nonportable'))

        self.assertIsInstance(html_module.data, basestring)
        new_data = html_module.data = html_module.data.replace('/static/', '/c4x/{0}/{1}/asset/'.format(
            source_course_id.org, source_course_id.run))
        module_store.update_item(html_module, self.user.id)

        html_module = module_store.get_item(html_module.location)
        self.assertEqual(new_data, html_module.data)

        # create the destination course
        _create_course(self, dest_course_id, course_data)

        # do the actual cloning
        clone_course(module_store, content_store, source_course_id, dest_course_id)

        # make sure that any non-portable links are rewritten during cloning
        html_module = module_store.get_item(dest_course_id.make_usage_key('html', 'nonportable'))

        self.assertIn('/asset/foo.jpg', html_module.data)

    def test_illegal_draft_crud_ops(self):
        draft_store = modulestore('draft')

        course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')

        location = course.id.make_usage_key('chapter', 'neuvo')
        # Ensure draft mongo store does not create drafts for things that shouldn't be draft
        newobject = draft_store.create_and_save_xmodule(location)
        self.assertFalse(getattr(newobject, 'is_draft', False))
        with self.assertRaises(InvalidVersionError):
            draft_store.convert_to_draft(location)
        chapter = draft_store.get_item(location)
        chapter.data = 'chapter data'

        draft_store.update_item(chapter, self.user.id)
        newobject = draft_store.get_item(chapter.location)
        self.assertFalse(getattr(newobject, 'is_draft', False))

        with self.assertRaises(InvalidVersionError):
            draft_store.unpublish(location)

    def test_bad_contentstore_request(self):
        resp = self.client.get_html('http://localhost:8001/c4x/CDX/123123/asset/&images_circuits_Lab7Solution2.png')
        self.assertEqual(resp.status_code, 400)
        _test_no_locations(self, resp, 400)

    def test_rewrite_nonportable_links_on_import(self):
        module_store = modulestore('direct')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)

        # first check a static asset link
        course_key = SlashSeparatedCourseKey('edX', 'toy', 'run')
        html_module_location = course_key.make_usage_key('html', 'nonportable')
        html_module = module_store.get_item(html_module_location)
        self.assertIn('/static/foo.jpg', html_module.data)

        # then check a intra courseware link
        html_module_location = course_key.make_usage_key('html', 'nonportable_link')
        html_module = module_store.get_item(html_module_location)
        self.assertIn('/jump_to_id/nonportable_link', html_module.data)

    def test_delete_course(self):
        """
        This test will import a course, make a draft item, and delete it. This will also assert that the
        draft content is also deleted
        """
        module_store = modulestore('direct')

        content_store = contentstore()
        draft_store = modulestore('draft')

        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)

        course_id = course_items[0].id

        # get a vertical (and components in it) to put into 'draft'
        vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)

        draft_store.convert_to_draft(vertical.location)
        for child in vertical.get_children():
            draft_store.convert_to_draft(child.location)

        # delete the course
        delete_course(module_store, content_store, course_id, commit=True)

        # assert that there's absolutely no non-draft modules in the course
        # this should also include all draft items
        items = module_store.get_items(course_id)
        self.assertEqual(len(items), 0)

        # assert that all content in the asset library is also deleted
        assets, count = content_store.get_all_content_for_course(course_id)
        self.assertEqual(len(assets), 0)
        self.assertEqual(count, 0)

    def verify_content_existence(self, store, root_dir, course_id, dirname, category_name, filename_suffix=''):
        filesystem = OSFS(root_dir / 'test_export')
        self.assertTrue(filesystem.exists(dirname))

        items = store.get_items(course_id, category=category_name)

        for item in items:
            filesystem = OSFS(root_dir / ('test_export/' + dirname))
            self.assertTrue(filesystem.exists(item.location.name + filename_suffix))

    @mock.patch('xmodule.course_module.requests.get')
    def test_export_course(self, mock_get):
        mock_get.return_value.text = dedent("""
            <?xml version="1.0"?><table_of_contents>
            <entry page="5" page_label="ii" name="Table of Contents"/>
            </table_of_contents>
        """).strip()

        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        # get a vertical (and components in it) to copy into an orphan sub dag
        vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
        # We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
        vertical.location = mongo.draft.as_draft(vertical.location.replace(name='no_references'))

        draft_store.update_item(vertical, allow_not_found=True)
        orphan_vertical = draft_store.get_item(vertical.location)
        self.assertEqual(orphan_vertical.location.name, 'no_references')

        # get the original vertical (and components in it) to put into 'draft'
        vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
        self.assertEqual(len(orphan_vertical.children), len(vertical.children))
        draft_store.convert_to_draft(vertical.location)
        for child in vertical.get_children():
            draft_store.convert_to_draft(child.location)

        root_dir = path(mkdtemp_clean())

        # now create a new/different private (draft only) vertical
        vertical.location = mongo.draft.as_draft(course_id.make_usage_key('vertical', 'a_private_vertical'))
        draft_store.update_item(vertical, allow_not_found=True)
        private_vertical = draft_store.get_item(vertical.location)
        vertical = None  # blank out b/c i destructively manipulated its location 2 lines above

        # add the new private to list of children
        sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))
        private_location_no_draft = private_vertical.location.replace(revision=None)
        sequential.children.append(private_location_no_draft)
        module_store.update_item(sequential, self.user.id)

        # read back the sequential, to make sure we have a pointer to
        sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))

        self.assertIn(private_location_no_draft, sequential.children)

        locked_asset_key = self._lock_an_asset(content_store, course_id)
        locked_asset_attrs = content_store.get_attrs(locked_asset_key)
        # the later import will reupload
        del locked_asset_attrs['uploadDate']

        print 'Exporting to tempdir = {0}'.format(root_dir)

        # export out to a tempdir
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)

        # check for static tabs
        self.verify_content_existence(module_store, root_dir, course_id, 'tabs', 'static_tab', '.html')

        # check for about content
        self.verify_content_existence(module_store, root_dir, course_id, 'about', 'about', '.html')

        # check for grading_policy.json
        filesystem = OSFS(root_dir / 'test_export/policies/2012_Fall')
        self.assertTrue(filesystem.exists('grading_policy.json'))

        course = module_store.get_course(course_id)
        # compare what's on disk compared to what we have in our course
        with filesystem.open('grading_policy.json', 'r') as grading_policy:
            on_disk = loads(grading_policy.read())
            self.assertEqual(on_disk, course.grading_policy)

        # check for policy.json
        self.assertTrue(filesystem.exists('policy.json'))

        # compare what's on disk to what we have in the course module
        with filesystem.open('policy.json', 'r') as course_policy:
            on_disk = loads(course_policy.read())
            self.assertIn('course/2012_Fall', on_disk)
            self.assertEqual(on_disk['course/2012_Fall'], own_metadata(course))

        # remove old course
        delete_course(module_store, content_store, course_id, commit=True)
        # reimport over old course
        self.check_import(
            module_store, root_dir, draft_store, content_store, course_id,
            locked_asset_key, locked_asset_attrs
        )
        # import to different course id
        self.check_import(
            module_store, root_dir, draft_store, content_store, SlashSeparatedCourseKey('anotherX', 'anotherToy', 'Someday'),
            locked_asset_key, locked_asset_attrs
        )

        shutil.rmtree(root_dir)

    def check_import(self, module_store, root_dir, draft_store, content_store, course_id,
                     locked_asset_key, locked_asset_attrs):
        # reimport
        import_from_xml(
            module_store,
            root_dir,
            ['test_export'],
            draft_store=draft_store,
            static_content_store=content_store,
            target_course_id=course_id,
        )

        items = module_store.get_items(course_id, category='vertical')
        self._check_verticals(items)

        # verify that we have the content in the draft store as well
        vertical = draft_store.get_item(
            course_id.make_usage_key('vertical', 'vertical_test'),
            depth=1
        )

        self.assertTrue(getattr(vertical, 'is_draft', False))
        self.assertNotIn('index_in_children_list', vertical.xml_attributes)
        self.assertNotIn('parent_sequential_url', vertical.xml_attributes)

        for child in vertical.get_children():
            self.assertTrue(getattr(child, 'is_draft', False))
            self.assertNotIn('index_in_children_list', child.xml_attributes)
            if hasattr(child, 'data'):
                self.assertNotIn('index_in_children_list', child.data)
            self.assertNotIn('parent_sequential_url', child.xml_attributes)
            if hasattr(child, 'data'):
                self.assertNotIn('parent_sequential_url', child.data)

        # make sure that we don't have a sequential that is in draft mode
        sequential = draft_store.get_item(
            course_id.make_usage_key('sequential', 'vertical_sequential')
        )

        self.assertFalse(getattr(sequential, 'is_draft', False))

        # verify that we have the private vertical
        test_private_vertical = draft_store.get_item(
            course_id.make_usage_key('vertical', 'a_private_vertical')
        )

        self.assertTrue(getattr(test_private_vertical, 'is_draft', False))

        # make sure the textbook survived the export/import
        course = module_store.get_course(course_id)

        self.assertGreater(len(course.textbooks), 0)

        locked_asset_key = locked_asset_key.map_into_course(course_id)
        new_attrs = content_store.get_attrs(locked_asset_key)
        for key, value in locked_asset_attrs.iteritems():
            if key == '_id':
                self.assertEqual(value['name'], new_attrs[key]['name'])
            elif key == 'filename':
                pass
            else:
                self.assertEqual(value, new_attrs[key])

    def test_export_course_with_metadata_only_video(self):
        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        # create a new video module and add it as a child to a vertical
        # this re-creates a bug whereby since the video template doesn't have
        # anything in 'data' field, the export was blowing up
        verticals = module_store.get_items(course_id, category='vertical')

        self.assertGreater(len(verticals), 0)

        parent = verticals[0]

        ItemFactory.create(parent_location=parent.location, category="video", display_name="untitled")

        root_dir = path(mkdtemp_clean())

        print 'Exporting to tempdir = {0}'.format(root_dir)

        # export out to a tempdir
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)

        shutil.rmtree(root_dir)

    def test_export_course_with_metadata_only_word_cloud(self):
        """
        Similar to `test_export_course_with_metadata_only_video`.
        """
        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['word_cloud'])
        course_id = SlashSeparatedCourseKey('HarvardX', 'ER22x', '2013_Spring')

        verticals = module_store.get_items(course_id, category='vertical')

        self.assertGreater(len(verticals), 0)

        parent = verticals[0]

        ItemFactory.create(parent_location=parent.location, category="word_cloud", display_name="untitled")

        root_dir = path(mkdtemp_clean())

        print 'Exporting to tempdir = {0}'.format(root_dir)

        # export out to a tempdir
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)

        shutil.rmtree(root_dir)

    def test_empty_data_roundtrip(self):
        """
        Test that an empty `data` field is preserved through
        export/import.
        """
        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        verticals = module_store.get_items(course_id, category='vertical')

        self.assertGreater(len(verticals), 0)

        parent = verticals[0]

        # Create a module, and ensure that its `data` field is empty
        word_cloud = ItemFactory.create(parent_location=parent.location, category="word_cloud", display_name="untitled")
        del word_cloud.data
        self.assertEquals(word_cloud.data, '')

        # Export the course
        root_dir = path(mkdtemp_clean())
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip', draft_modulestore=draft_store)

        # Reimport and get the video back
        import_from_xml(module_store, root_dir)
        imported_word_cloud = module_store.get_item(course_id.make_usage_key('word_cloud', 'untitled'))

        # It should now contain empty data
        self.assertEquals(imported_word_cloud.data, '')

    def test_html_export_roundtrip(self):
        """
        Test that a course which has HTML that has style formatting is preserved in export/import
        """
        module_store = modulestore('direct')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])

        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        # Export the course
        root_dir = path(mkdtemp_clean())
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip')

        # Reimport and get the video back
        import_from_xml(module_store, root_dir)

        # get the sample HTML with styling information
        html_module = module_store.get_item(course_id.make_usage_key('html', 'with_styling'))
        self.assertIn('<p style="font:italic bold 72px/30px Georgia, serif; color: red; ">', html_module.data)

        # get the sample HTML with just a simple <img> tag information
        html_module = module_store.get_item(course_id.make_usage_key('html', 'just_img'))
        self.assertIn('<img src="/static/foo_bar.jpg" />', html_module.data)

    def test_course_handouts_rewrites(self):
        module_store = modulestore('direct')

        # import a test course
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = course_items[0].id

        handouts_location = course_id.make_usage_key('course_info', 'handouts')

        # get module info (json)
        resp = self.client.get(get_url('xblock_handler', handouts_location))

        # make sure we got a successful response
        self.assertEqual(resp.status_code, 200)
        # check that /static/ has been converted to the full path
        # note, we know the link it should be because that's what in the 'toy' course in the test data
        self.assertContains(resp, '/c4x/edX/toy/asset/handouts_sample_handout.txt')

    def test_prefetch_children(self):
        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        wrapper = MongoCollectionFindWrapper(module_store.collection.find)
        module_store.collection.find = wrapper.find
        print module_store.metadata_inheritance_cache_subsystem
        print module_store.request_cache
        course = module_store.get_course(course_id, depth=2)

        # make sure we haven't done too many round trips to DB
        # note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials
        # Because we're querying from the top of the tree, we cache information needed for inheritance,
        # so we don't need to make an extra query to compute it.
        self.assertEqual(wrapper.counter, 3)

        # make sure we pre-fetched a known sequential which should be at depth=2
        self.assertTrue(course_id.make_usage_key('sequential', 'vertical_sequential') in course.system.module_data)

        # make sure we don't have a specific vertical which should be at depth=3
        self.assertFalse(course_id.make_usage_key('vertical', 'vertical_test') in course.system.module_data)

    def test_export_course_without_content_store(self):
        module_store = modulestore('direct')
        content_store = contentstore()

        # Create toy course

        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = course_items[0].id

        root_dir = path(mkdtemp_clean())

        print 'Exporting to tempdir = {0}'.format(root_dir)
        export_to_xml(module_store, None, course_id, root_dir, 'test_export_no_content_store')

        # Delete the course from module store and reimport it

        delete_course(module_store, content_store, course_id, commit=True)

        import_from_xml(
            module_store, root_dir, ['test_export_no_content_store'],
            draft_store=None,
            static_content_store=None,
            target_course_id=course_id
        )

        # Verify reimported course

        items = module_store.get_items(
            course_id,
            category='sequential',
            name='vertical_sequential'
        )
        self.assertEqual(len(items), 1)

    def _check_verticals(self, items):
        """ Test getting the editing HTML for each vertical. """
        # Assert is here to make sure that the course being tested actually has verticals (units) to check.
        self.assertGreater(len(items), 0)
        for descriptor in items:
            resp = self.client.get_html(get_url('unit_handler', descriptor.location))
            self.assertEqual(resp.status_code, 200)
            _test_no_locations(self, resp)


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreTest(ModuleStoreTestCase):
    """
    Tests for the CMS ContentStore application.
    """
    def setUp(self):
        """
        These tests need a user in the DB so that the django Test Client
        can log them in.
        They inherit from the ModuleStoreTestCase class so that the mongodb collection
        will be cleared out before each test case execution and deleted
        afterwards.
        """
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

        self.course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
            'run': '2013_Spring'
        }

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def assert_created_course(self, number_suffix=None):
        """
        Checks that the course was created properly.
        """
        test_course_data = {}
        test_course_data.update(self.course_data)
        if number_suffix:
            test_course_data['number'] = '{0}_{1}'.format(test_course_data['number'], number_suffix)
        course_key = _get_course_id(test_course_data)
        _create_course(self, course_key, test_course_data)
        # Verify that the creator is now registered in the course.
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_key))
        return test_course_data

    def assert_create_course_failed(self, error_message):
        """
        Checks that the course not created.
        """
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 400)
        data = parse_json(resp)
        self.assertEqual(data['error'], error_message)

    def test_create_course(self):
        """Test new course creation - happy path"""
        self.assert_created_course()

    def test_create_course_with_dots(self):
        """Test new course creation with dots in the name"""
        self.course_data['org'] = 'org.foo.bar'
        self.course_data['number'] = 'course.number'
        self.course_data['run'] = 'run.name'
        self.assert_created_course()

    def test_create_course_check_forum_seeding(self):
        """Test new course creation and verify forum seeding """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        self.assertTrue(are_permissions_roles_seeded(_get_course_id(test_course_data)))

    def test_forum_unseeding_on_delete(self):
        """Test new course creation and verify forum unseeding """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        course_id = _get_course_id(test_course_data)
        self.assertTrue(are_permissions_roles_seeded(course_id))
        delete_course_and_groups(course_id, commit=True)
        # should raise an exception for checking permissions on deleted course
        with self.assertRaises(ItemNotFoundError):
            are_permissions_roles_seeded(course_id)

    def test_forum_unseeding_with_multiple_courses(self):
        """Test new course creation and verify forum unseeding when there are multiple courses"""
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        second_course_data = self.assert_created_course(number_suffix=uuid4().hex)

        # unseed the forums for the first course
        course_id = _get_course_id(test_course_data)
        delete_course_and_groups(course_id, commit=True)
        # should raise an exception for checking permissions on deleted course
        with self.assertRaises(ItemNotFoundError):
            are_permissions_roles_seeded(course_id)

        second_course_id = _get_course_id(second_course_data)
        # permissions should still be there for the other course
        self.assertTrue(are_permissions_roles_seeded(second_course_id))

    def test_course_enrollments_and_roles_on_delete(self):
        """
        Test that course deletion doesn't remove course enrollments or user's roles
        """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        course_id = _get_course_id(test_course_data)

        # test that a user gets his enrollment and its 'student' role as default on creating a course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_id))
        self.assertTrue(self.user.roles.filter(name="Student", course_id=course_id))  # pylint: disable=no-member

        delete_course_and_groups(course_id, commit=True)
        # check that user's enrollment for this course is not deleted
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_id))
        # check that user has form role "Student" for this course even after deleting it
        self.assertTrue(self.user.roles.filter(name="Student", course_id=course_id))  # pylint: disable=no-member

    def test_course_access_groups_on_delete(self):
        """
        Test that course deletion removes users from 'instructor' and 'staff' groups of this course
        of all format e.g, 'instructor_edX/Course/Run', 'instructor_edX.Course.Run', 'instructor_Course'
        """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        course_id = _get_course_id(test_course_data)

        # Add user in possible groups and check that user in instructor groups of this course
        instructor_role = CourseInstructorRole(course_id)

        auth.add_users(self.user, instructor_role, self.user)

        self.assertTrue(len(instructor_role.users_with_role()) > 0)

        # Now delete course and check that user not in instructor groups of this course
        delete_course_and_groups(course_id, commit=True)

        # Update our cached user since its roles have changed
        self.user = User.objects.get_by_natural_key(self.user.natural_key()[0])

        self.assertFalse(instructor_role.has_user(self.user))
        self.assertEqual(len(instructor_role.users_with_role()), 0)

    def test_create_course_duplicate_course(self):
        """Test new course creation - error path"""
        self.client.ajax_post('/course/', self.course_data)
        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')

    def assert_course_creation_failed(self, error_message):
        """
        Checks that the course did not get created
        """
        test_enrollment = False
        try:
            course_id = _get_course_id(self.course_data)
            initially_enrolled = CourseEnrollment.is_enrolled(self.user, course_id)
            test_enrollment = True
        except InvalidKeyError:
            # b/c the intent of the test with bad chars isn't to test auth but to test the handler, ignore
            pass
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertRegexpMatches(data['ErrMsg'], error_message)
        if test_enrollment:
            # One test case involves trying to create the same course twice. Hence for that course,
            # the user will be enrolled. In the other cases, initially_enrolled will be False.
            self.assertEqual(initially_enrolled, CourseEnrollment.is_enrolled(self.user, course_id))

    def test_create_course_duplicate_number(self):
        """Test new course creation - error path"""
        self.client.ajax_post('/course/', self.course_data)
        self.course_data['display_name'] = 'Robot Super Course Two'
        self.course_data['run'] = '2013_Summer'

        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')

    def test_create_course_case_change(self):
        """Test new course creation - error path due to case insensitive name equality"""
        self.course_data['number'] = 'capital'
        self.client.ajax_post('/course/', self.course_data)
        cache_current = self.course_data['org']
        self.course_data['org'] = self.course_data['org'].lower()
        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')
        self.course_data['org'] = cache_current

        self.client.ajax_post('/course/', self.course_data)
        cache_current = self.course_data['number']
        self.course_data['number'] = self.course_data['number'].upper()
        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')

    def test_course_substring(self):
        """
        Test that a new course can be created whose name is a substring of an existing course
        """
        self.client.ajax_post('/course/', self.course_data)
        cache_current = self.course_data['number']
        self.course_data['number'] = '{}a'.format(self.course_data['number'])
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 200)
        self.course_data['number'] = cache_current
        self.course_data['org'] = 'a{}'.format(self.course_data['org'])
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 200)

    def test_create_course_with_bad_organization(self):
        """Test new course creation - error path for bad organization name"""
        self.course_data['org'] = 'University of California, Berkeley'
        self.assert_course_creation_failed(
            r"(?s)Unable to create course 'Robot Super Course'.*: Invalid characters in u'University of California, Berkeley'")

    def test_create_course_with_course_creation_disabled_staff(self):
        """Test new course creation -- course creation disabled, but staff access."""
        with mock.patch.dict('django.conf.settings.FEATURES', {'DISABLE_COURSE_CREATION': True}):
            self.assert_created_course()

    def test_create_course_with_course_creation_disabled_not_staff(self):
        """Test new course creation -- error path for course creation disabled, not staff access."""
        with mock.patch.dict('django.conf.settings.FEATURES', {'DISABLE_COURSE_CREATION': True}):
            self.user.is_staff = False
            self.user.save()
            self.assert_course_permission_denied()

    def test_create_course_no_course_creators_staff(self):
        """Test new course creation -- course creation group enabled, staff, group is empty."""
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}):
            self.assert_created_course()

    def test_create_course_no_course_creators_not_staff(self):
        """Test new course creation -- error path for course creator group enabled, not staff, group is empty."""
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            self.user.is_staff = False
            self.user.save()
            self.assert_course_permission_denied()

    def test_create_course_with_course_creator(self):
        """Test new course creation -- use course creator group"""
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            auth.add_users(self.user, CourseCreatorRole(), self.user)
            self.assert_created_course()

    def test_create_course_with_unicode_in_id_disabled(self):
        """
        Test new course creation with feature setting: ALLOW_UNICODE_COURSE_ID disabled.
        """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ALLOW_UNICODE_COURSE_ID': False}):
            error_message = "Special characters not allowed in organization, course number, and course run."
            self.course_data['org'] = u'��������������'
            self.assert_create_course_failed(error_message)

            self.course_data['number'] = u'��chantillon'
            self.assert_create_course_failed(error_message)

            self.course_data['run'] = u'����������'
            self.assert_create_course_failed(error_message)

    def assert_course_permission_denied(self):
        """
        Checks that the course did not get created due to a PermissionError.
        """
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 403)

    def test_course_index_view_with_no_courses(self):
        """Test viewing the index page with no courses"""
        # Create a course so there is something to view
        resp = self.client.get_html('/course/')
        self.assertContains(
            resp,
            '<h1 class="page-header">My Courses</h1>',
            status_code=200,
            html=True
        )
        _test_no_locations(self, resp)

    def test_course_factory(self):
        """Test that the course factory works correctly."""
        course = CourseFactory.create()
        self.assertIsInstance(course, CourseDescriptor)

    def test_item_factory(self):
        """Test that the item factory works correctly."""
        course = CourseFactory.create()
        item = ItemFactory.create(parent_location=course.location)
        self.assertIsInstance(item, SequenceDescriptor)

    def test_course_index_view_with_course(self):
        """Test viewing the index page with an existing course"""
        CourseFactory.create(display_name='Robot Super Educational Course')
        resp = self.client.get_html('/course/')
        self.assertContains(
            resp,
            '<h3 class="course-title">Robot Super Educational Course</h3>',
            status_code=200,
            html=True
        )
        _test_no_locations(self, resp)

    def test_course_overview_view_with_course(self):
        """Test viewing the course overview page with an existing course"""
        course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
        resp = self._show_course_overview(course.id)
        self.assertContains(
            resp,
            '<article class="courseware-overview" data-locator="location:MITx+999+Robot_Super_Course+course+Robot_Super_Course" data-course-key="slashes:MITx+999+Robot_Super_Course">',
            status_code=200,
            html=True
        )

    def test_create_item(self):
        """Test creating a new xblock instance."""
        course = _course_factory_create_course()

        section_data = {
            'parent_locator': unicode(course.location),
            'category': 'chapter',
            'display_name': 'Section One',
        }

        resp = self.client.ajax_post(reverse_url('xblock_handler'), section_data)
        _test_no_locations(self, resp, html=False)

        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertRegexpMatches(
            data['locator'],
            r"location:MITx\+999\+Robot_Super_Course\+chapter\+([0-9]|[a-f]){3,}$"
        )

    def test_capa_module(self):
        """Test that a problem treats markdown specially."""
        course = _course_factory_create_course()

        problem_data = {
            'parent_locator': unicode(course.location),
            'category': 'problem'
        }

        resp = self.client.ajax_post(reverse_url('xblock_handler'), problem_data)
        self.assertEqual(resp.status_code, 200)
        payload = parse_json(resp)
        problem_loc = UsageKey.from_string(payload['locator'])
        problem = get_modulestore(problem_loc).get_item(problem_loc)
        # should be a CapaDescriptor
        self.assertIsInstance(problem, CapaDescriptor, "New problem is not a CapaDescriptor")
        context = problem.get_context()
        self.assertIn('markdown', context, "markdown is missing from context")
        self.assertNotIn('markdown', problem.editable_metadata_fields, "Markdown slipped into the editable metadata fields")

    def test_cms_imported_course_walkthrough(self):
        """
        Import and walk through some common URL endpoints. This just verifies non-500 and no other
        correct behavior, so it is not a deep test
        """
        def test_get_html(handler):
            # Helper function for getting HTML for a page in Studio and
            # checking that it does not error.
            resp = self.client.get_html(
                get_url(handler, course_key, 'course_key_string')
            )
            self.assertEqual(resp.status_code, 200)
            _test_no_locations(self, resp)

        _, course_items = import_from_xml(modulestore('direct'), 'common/test/data/', ['simple'])
        course_key = course_items[0].id

        resp = self._show_course_overview(course_key)
        self.assertEqual(resp.status_code, 200)
        self.assertContains(resp, 'Chapter 2')

        # go to various pages
        test_get_html('import_handler')
        test_get_html('export_handler')
        test_get_html('course_team_handler')
        test_get_html('course_info_handler')
        test_get_html('checklists_handler')
        test_get_html('assets_handler')
        test_get_html('tabs_handler')
        test_get_html('settings_handler')
        test_get_html('grading_handler')
        test_get_html('advanced_settings_handler')
        test_get_html('textbooks_list_handler')

        # go look at a subsection page
        subsection_key = course_key.make_usage_key('sequential', 'test_sequence')
        resp = self.client.get_html(get_url('subsection_handler', subsection_key))
        self.assertEqual(resp.status_code, 200)
        _test_no_locations(self, resp)

        # go look at the Edit page
        unit_key = course_key.make_usage_key('vertical', 'test_vertical')
        resp = self.client.get_html(get_url('unit_handler', unit_key))
        self.assertEqual(resp.status_code, 200)
        _test_no_locations(self, resp)

        def delete_item(category, name):
            """ Helper method for testing the deletion of an xblock item. """
            item_key = course_key.make_usage_key(category, name)
            resp = self.client.delete(get_url('xblock_handler', item_key))
            self.assertEqual(resp.status_code, 204)
            _test_no_locations(self, resp, status_code=204, html=False)

        # delete a component
        delete_item(category='html', name='test_html')

        # delete a unit
        delete_item(category='vertical', name='test_vertical')

        # delete a unit
        delete_item(category='sequential', name='test_sequence')

        # delete a chapter
        delete_item(category='chapter', name='chapter_2')

    def test_import_into_new_course_id(self):
        module_store = modulestore('direct')
        target_course_id = _get_course_id(self.course_data)
        _create_course(self, target_course_id, self.course_data)

        import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)

        modules = module_store.get_items(target_course_id)

        # we should have a number of modules in there
        # we can't specify an exact number since it'll always be changing
        self.assertGreater(len(modules), 10)

        #
        # test various re-namespacing elements
        #

        # first check PDF textbooks, to make sure the url paths got updated
        course_module = module_store.get_course(target_course_id)

        self.assertEqual(len(course_module.pdf_textbooks), 1)
        self.assertEqual(len(course_module.pdf_textbooks[0]["chapters"]), 2)
        self.assertEqual(course_module.pdf_textbooks[0]["chapters"][0]["url"], '/static/Chapter1.pdf')
        self.assertEqual(course_module.pdf_textbooks[0]["chapters"][1]["url"], '/static/Chapter2.pdf')

    def test_import_into_new_course_id_wiki_slug_renamespacing(self):
        module_store = modulestore('direct')

        # If reimporting into the same course do not change the wiki_slug.
        target_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
        course_data = {
            'org': target_course_id.org,
            'number': target_course_id.course,
            'display_name': 'Robot Super Course',
            'run': target_course_id.run
        }
        _create_course(self, target_course_id, course_data)
        course_module = module_store.get_course(target_course_id)
        course_module.wiki_slug = 'toy'
        course_module.save()

        # Import a course with wiki_slug == location.course
        import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
        course_module = module_store.get_course(target_course_id)
        self.assertEquals(course_module.wiki_slug, 'toy')

        # But change the wiki_slug if it is a different course.
        target_course_id = SlashSeparatedCourseKey('MITx', '999', '2013_Spring')
        course_data = {
            'org': target_course_id.org,
            'number': target_course_id.course,
            'display_name': 'Robot Super Course',
            'run': target_course_id.run
        }
        _create_course(self, target_course_id, course_data)

        # Import a course with wiki_slug == location.course
        import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
        course_module = module_store.get_course(target_course_id)
        self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')

        # Now try importing a course with wiki_slug == '{0}.{1}.{2}'.format(location.org, location.course, location.run)
        import_from_xml(module_store, 'common/test/data/', ['two_toys'], target_course_id=target_course_id)
        course_module = module_store.get_course(target_course_id)
        self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')

    def test_import_metadata_with_attempts_empty_string(self):
        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['simple'])
        did_load_item = False
        try:
            course_key = SlashSeparatedCourseKey('edX', 'simple', 'problem')
            usage_key = course_key.make_usage_key('problem', 'ps01-simple')
            module_store.get_item(usage_key)
            did_load_item = True
        except ItemNotFoundError:
            pass

        # make sure we found the item (e.g. it didn't error while loading)
        self.assertTrue(did_load_item)

    def test_forum_id_generation(self):
        module_store = modulestore('direct')
        course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
        new_component_location = course.id.make_usage_key('discussion', 'new_component')

        # crate a new module and add it as a child to a vertical
        module_store.create_and_save_xmodule(new_component_location)

        new_discussion_item = module_store.get_item(new_component_location)

        self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$')

    def test_metadata_inheritance(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])

        course = course_items[0]
        verticals = module_store.get_items(course.id, category='vertical')

        # let's assert on the metadata_inheritance on an existing vertical
        for vertical in verticals:
            self.assertEqual(course.xqa_key, vertical.xqa_key)
            self.assertEqual(course.start, vertical.start)

        self.assertGreater(len(verticals), 0)

        new_component_location = course.id.make_usage_key('html', 'new_component')

        # crate a new module and add it as a child to a vertical
        module_store.create_and_save_xmodule(new_component_location)
        parent = verticals[0]
        parent.children.append(new_component_location)
        module_store.update_item(parent, self.user.id)

        # flush the cache
        module_store.refresh_cached_metadata_inheritance_tree(new_component_location.course_key)
        new_module = module_store.get_item(new_component_location)

        # check for grace period definition which should be defined at the course level
        self.assertEqual(parent.graceperiod, new_module.graceperiod)
        self.assertEqual(parent.start, new_module.start)
        self.assertEqual(course.start, new_module.start)

        self.assertEqual(course.xqa_key, new_module.xqa_key)

        #
        # now let's define an override at the leaf node level
        #
        new_module.graceperiod = timedelta(1)
        module_store.update_item(new_module, self.user.id)

        # flush the cache and refetch
        module_store.refresh_cached_metadata_inheritance_tree(new_component_location.course_key)
        new_module = module_store.get_item(new_component_location)

        self.assertEqual(timedelta(1), new_module.graceperiod)

    def test_default_metadata_inheritance(self):
        course = CourseFactory.create()
        vertical = ItemFactory.create(parent_location=course.location)
        course.children.append(vertical)
        # in memory
        self.assertIsNotNone(course.start)
        self.assertEqual(course.start, vertical.start)
        self.assertEqual(course.textbooks, [])
        self.assertIn('GRADER', course.grading_policy)
        self.assertIn('GRADE_CUTOFFS', course.grading_policy)
        self.assertGreaterEqual(len(course.checklists), 4)

        # by fetching
        module_store = modulestore('direct')
        fetched_course = module_store.get_item(course.location)
        fetched_item = module_store.get_item(vertical.location)
        self.assertIsNotNone(fetched_course.start)
        self.assertEqual(course.start, fetched_course.start)
        self.assertEqual(fetched_course.start, fetched_item.start)
        self.assertEqual(course.textbooks, fetched_course.textbooks)
        # is this test too strict? i.e., it requires the dicts to be ==
        self.assertEqual(course.checklists, fetched_course.checklists)

    def test_image_import(self):
        """Test backwards compatibilty of course image."""
        module_store = modulestore('direct')

        content_store = contentstore()

        # Use conditional_and_poll, as it's got an image already
        import_from_xml(
            module_store,
            'common/test/data/',
            ['conditional_and_poll'],
            static_content_store=content_store
        )

        course = module_store.get_courses()[0]

        # Make sure the course image is set to the right place
        self.assertEqual(course.course_image, 'images_course_image.jpg')

        # Ensure that the imported course image is present -- this shouldn't raise an exception
        asset_key = course.id.make_asset_key('asset', course.course_image)
        content_store.find(asset_key)

    def _show_course_overview(self, course_key):
        """
        Show the course overview page.
        """
        resp = self.client.get_html(get_url('course_handler', course_key, 'course_key_string'))
        _test_no_locations(self, resp)
        return resp

    def test_wiki_slug(self):
        """When creating a course a unique wiki_slug should be set."""

        course_key = _get_course_id(self.course_data)
        _create_course(self, course_key, self.course_data)
        course_module = modulestore('direct').get_course(course_key)
        self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')


@override_settings(MODULESTORE=TEST_MODULESTORE)
class MetadataSaveTestCase(ModuleStoreTestCase):
    """Test that metadata is correctly cached and decached."""

    def setUp(self):
        course = CourseFactory.create(
            org='edX', course='999', display_name='Robot Super Course')

        video_sample_xml = '''
        <video display_name="Test Video"
                youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
                show_captions="false"
                from="00:00:01"
                to="00:01:00">
            <source src="http://www.example.com/file.mp4"/>
            <track src="http://www.example.com/track"/>
        </video>
        '''
        self.video_descriptor = ItemFactory.create(
            parent_location=course.location, category='video',
            data={'data': video_sample_xml}
        )

    def test_metadata_not_persistence(self):
        """
        Test that descriptors which set metadata fields in their
        constructor are correctly deleted.
        """
        self.assertIn('html5_sources', own_metadata(self.video_descriptor))
        attrs_to_strip = {
            'show_captions',
            'youtube_id_1_0',
            'youtube_id_0_75',
            'youtube_id_1_25',
            'youtube_id_1_5',
            'start_time',
            'end_time',
            'source',
            'html5_sources',
            'track'
        }

        location = self.video_descriptor.location

        for field_name in attrs_to_strip:
            delattr(self.video_descriptor, field_name)

        self.assertNotIn('html5_sources', own_metadata(self.video_descriptor))
        get_modulestore(location).update_item(self.video_descriptor, '**replace_user**')
        module = get_modulestore(location).get_item(location)

        self.assertNotIn('html5_sources', own_metadata(module))

    def test_metadata_persistence(self):
        # TODO: create the same test as `test_metadata_not_persistence`,
        # but check persistence for some other module.
        pass


class EntryPageTestCase(TestCase):
    """
    Tests entry pages that aren't specific to a course.
    """
    def setUp(self):
        self.client = AjaxEnabledTestClient()

    def _test_page(self, page, status_code=200):
        resp = self.client.get_html(page)
        self.assertEqual(resp.status_code, status_code)
        _test_no_locations(self, resp, status_code)

    def test_how_it_works(self):
        self._test_page("/howitworks")

    def test_signup(self):
        self._test_page("/signup")

    def test_login(self):
        self._test_page("/signin")

    def test_logout(self):
        # Logout redirects.
        self._test_page("/logout", 302)


def _create_course(test, course_key, course_data):
    """
    Creates a course via an AJAX request and verifies the URL returned in the response.
    """
    course_url = get_url('course_handler', course_key, 'course_key_string')
    response = test.client.ajax_post(course_url, course_data)
    test.assertEqual(response.status_code, 200)
    data = parse_json(response)
    test.assertNotIn('ErrMsg', data)
    test.assertEqual(data['url'], course_url)


def _course_factory_create_course():
    """
    Creates a course via the CourseFactory and returns the locator for it.
    """
    return CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')


def _get_course_id(course_data):
    """Returns the course ID (org/number/run)."""
    return SlashSeparatedCourseKey(course_data['org'], course_data['number'], course_data['run'])


def _test_no_locations(test, resp, status_code=200, html=True):
    """
    Verifies that "i4x", which appears in old locations, but not
    new locators, does not appear in the HTML response output.
    Used to verify that database refactoring is complete.
    """
    test.assertNotContains(resp, 'i4x', status_code=status_code, html=html)
    if html:
        # For HTML pages, it is nice to call the method with html=True because
        # it checks that the HTML properly parses. However, it won't find i4x usages
        # in JavaScript blocks.
        content = resp.content
        hits = len(re.findall(r"(?<!jump_to/)i4x://", content))
        test.assertEqual(hits, 0, "i4x found outside of LMS jump-to links")

########NEW FILE########
__FILENAME__ = test_core_caching
from cache_toolbox.core import get_cached_content, set_cached_content, del_cached_content
from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent
from django.test import TestCase


class Content:
    def __init__(self, location, content):
        self.location = location
        self.content = content

    def get_id(self):
        return self.location.to_deprecated_son()


class CachingTestCase(TestCase):
#   Tests for https://edx.lighthouseapp.com/projects/102637/tickets/112-updating-asset-does-not-refresh-the-cached-copy
    unicodeLocation = Location(u'c4x', u'mitX', u'800', u'run', u'thumbnail', u'monsters.jpg')
    # Note that some of the parts are strings instead of unicode strings
    nonUnicodeLocation = Location('c4x', u'mitX', u'800', u'run', 'thumbnail', 'monsters.jpg')
    mockAsset = Content(unicodeLocation, 'my content')

    def test_put_and_get(self):
        set_cached_content(self.mockAsset)
        self.assertEqual(self.mockAsset.content, get_cached_content(self.unicodeLocation).content,
                         'should be stored in cache with unicodeLocation')
        self.assertEqual(self.mockAsset.content, get_cached_content(self.nonUnicodeLocation).content,
                         'should be stored in cache with nonUnicodeLocation')

    def test_delete(self):
        set_cached_content(self.mockAsset)
        del_cached_content(self.nonUnicodeLocation)
        self.assertEqual(None, get_cached_content(self.unicodeLocation),
                         'should not be stored in cache with unicodeLocation')
        self.assertEqual(None, get_cached_content(self.nonUnicodeLocation),
                         'should not be stored in cache with nonUnicodeLocation')

########NEW FILE########
__FILENAME__ = test_course_listing
"""
Unit tests for getting the list of courses for a user through iterating all courses and
by reversing group name formats.
"""
import random
from chrono import Timer

from django.contrib.auth.models import Group
from django.test import RequestFactory

from contentstore.views.course import _accessible_courses_list, _accessible_courses_list_from_groups
from contentstore.utils import delete_course_and_groups, reverse_course_url
from contentstore.tests.utils import AjaxEnabledTestClient
from student.tests.factories import UserFactory
from student.roles import CourseInstructorRole, CourseStaffRole
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.locations import SlashSeparatedCourseKey

TOTAL_COURSES_COUNT = 500
USER_COURSES_COUNT = 50


class TestCourseListing(ModuleStoreTestCase):
    """
    Unit tests for getting the list of courses for a logged in user
    """
    def setUp(self):
        """
        Add a user and a course
        """
        super(TestCourseListing, self).setUp()
        # create and log in a staff user.
        self.user = UserFactory(is_staff=True)  # pylint: disable=no-member
        self.factory = RequestFactory()
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.user.username, password='test')

    def _create_course_with_access_groups(self, course_location, user=None):
        """
        Create dummy course with 'CourseFactory' and role (instructor/staff) groups
        """
        course = CourseFactory.create(
            org=course_location.org,
            number=course_location.course,
            run=course_location.run
        )

        if user is not None:
            for role in [CourseInstructorRole, CourseStaffRole]:
                role(course.id).add_users(user)

        return course

    def tearDown(self):
        """
        Reverse the setup
        """
        self.client.logout()
        ModuleStoreTestCase.tearDown(self)

    def test_get_course_list(self):
        """
        Test getting courses with new access group format e.g. 'instructor_edx.course.run'
        """
        request = self.factory.get('/course/')
        request.user = self.user

        course_location = SlashSeparatedCourseKey('Org1', 'Course1', 'Run1')
        self._create_course_with_access_groups(course_location, self.user)

        # get courses through iterating all courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # get courses by reversing group name formats
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)
        # check both course lists have same courses
        self.assertEqual(courses_list, courses_list_by_groups)

    def test_get_course_list_with_invalid_course_location(self):
        """
        Test getting courses with invalid course location (course deleted from modulestore).
        """
        request = self.factory.get('/course')
        request.user = self.user

        course_key = SlashSeparatedCourseKey('Org', 'Course', 'Run')
        self._create_course_with_access_groups(course_key, self.user)

        # get courses through iterating all courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # get courses by reversing group name formats
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)
        # check both course lists have same courses
        self.assertEqual(courses_list, courses_list_by_groups)

        # now delete this course and re-add user to instructor group of this course
        delete_course_and_groups(course_key, commit=True)

        CourseInstructorRole(course_key).add_users(self.user)

        # test that get courses through iterating all courses now returns no course
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 0)

        # now test that get courses by reversing group name formats gives 'ItemNotFoundError'
        with self.assertRaises(ItemNotFoundError):
            _accessible_courses_list_from_groups(request)

    def test_course_listing_performance(self):
        """
        Create large number of courses and give access of some of these courses to the user and
        compare the time to fetch accessible courses for the user through traversing all courses and
        reversing django groups
        """
        # create and log in a non-staff user
        self.user = UserFactory()
        request = self.factory.get('/course')
        request.user = self.user
        self.client.login(username=self.user.username, password='test')

        # create list of random course numbers which will be accessible to the user
        user_course_ids = random.sample(range(TOTAL_COURSES_COUNT), USER_COURSES_COUNT)

        # create courses and assign those to the user which have their number in user_course_ids
        for number in range(TOTAL_COURSES_COUNT):
            org = 'Org{0}'.format(number)
            course = 'Course{0}'.format(number)
            run = 'Run{0}'.format(number)
            course_location = SlashSeparatedCourseKey(org, course, run)
            if number in user_course_ids:
                self._create_course_with_access_groups(course_location, self.user)
            else:
                self._create_course_with_access_groups(course_location)

        # time the get courses by iterating through all courses
        with Timer() as iteration_over_courses_time_1:
            courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # time again the get courses by iterating through all courses
        with Timer() as iteration_over_courses_time_2:
            courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # time the get courses by reversing django groups
        with Timer() as iteration_over_groups_time_1:
            courses_list = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # time again the get courses by reversing django groups
        with Timer() as iteration_over_groups_time_2:
            courses_list = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # test that the time taken by getting courses through reversing django groups is lower then the time
        # taken by traversing through all courses (if accessible courses are relatively small)
        self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed)
        self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed)

    def test_get_course_list_with_same_course_id(self):
        """
        Test getting courses with same id but with different name case. Then try to delete one of them and
        check that it is properly deleted and other one is accessible
        """
        # create and log in a non-staff user
        self.user = UserFactory()
        request = self.factory.get('/course')
        request.user = self.user
        self.client.login(username=self.user.username, password='test')

        course_location_caps = SlashSeparatedCourseKey('Org', 'COURSE', 'Run')
        self._create_course_with_access_groups(course_location_caps, self.user)

        # get courses through iterating all courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # get courses by reversing group name formats
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)
        # check both course lists have same courses
        self.assertEqual(courses_list, courses_list_by_groups)

        # now create another course with same course_id but different name case
        course_location_camel = SlashSeparatedCourseKey('Org', 'Course', 'Run')
        self._create_course_with_access_groups(course_location_camel, self.user)

        # test that get courses through iterating all courses returns both courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 2)

        # test that get courses by reversing group name formats returns both courses
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 2)

        # now delete first course (course_location_caps) and check that it is no longer accessible
        delete_course_and_groups(course_location_caps, commit=True)

        # test that get courses through iterating all courses now returns one course
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # test that get courses by reversing group name formats also returns one course
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)

        # now check that deleted course is not accessible
        outline_url = reverse_course_url('course_handler', course_location_caps)
        response = self.client.get(outline_url, HTTP_ACCEPT='application/json')
        self.assertEqual(response.status_code, 403)

        # now check that other course is accessible
        outline_url = reverse_course_url('course_handler', course_location_camel)
        response = self.client.get(outline_url, HTTP_ACCEPT='application/json')
        self.assertEqual(response.status_code, 200)

########NEW FILE########
__FILENAME__ = test_course_settings
"""
Tests for Studio Course Settings.
"""
import datetime
import json
import copy
import mock

from django.utils.timezone import UTC
from django.test.utils import override_settings

from models.settings.course_details import (CourseDetails, CourseSettingsEncoder)
from models.settings.course_grading import CourseGradingModel
from contentstore.utils import get_modulestore, EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
from xmodule.modulestore.tests.factories import CourseFactory

from models.settings.course_metadata import CourseMetadata
from xmodule.fields import Date

from .utils import CourseTestCase
from xmodule.modulestore.django import modulestore
from contentstore.views.component import ADVANCED_COMPONENT_POLICY_KEY


def get_url(course_id, handler_name='settings_handler'):
    return reverse_course_url(handler_name, course_id)

class CourseDetailsTestCase(CourseTestCase):
    """
    Tests the first course settings page (course dates, overview, etc.).
    """
    def test_virgin_fetch(self):
        details = CourseDetails.fetch(self.course.id)
        self.assertEqual(details.org, self.course.location.org, "Org not copied into")
        self.assertEqual(details.course_id, self.course.location.course, "Course_id not copied into")
        self.assertEqual(details.run, self.course.location.name, "Course name not copied into")
        self.assertEqual(details.course_image_name, self.course.course_image)
        self.assertIsNotNone(details.start_date.tzinfo)
        self.assertIsNone(details.end_date, "end date somehow initialized " + str(details.end_date))
        self.assertIsNone(details.enrollment_start, "enrollment_start date somehow initialized " + str(details.enrollment_start))
        self.assertIsNone(details.enrollment_end, "enrollment_end date somehow initialized " + str(details.enrollment_end))
        self.assertIsNone(details.syllabus, "syllabus somehow initialized" + str(details.syllabus))
        self.assertIsNone(details.intro_video, "intro_video somehow initialized" + str(details.intro_video))
        self.assertIsNone(details.effort, "effort somehow initialized" + str(details.effort))

    def test_encoder(self):
        details = CourseDetails.fetch(self.course.id)
        jsondetails = json.dumps(details, cls=CourseSettingsEncoder)
        jsondetails = json.loads(jsondetails)
        self.assertEqual(jsondetails['course_image_name'], self.course.course_image)
        self.assertIsNone(jsondetails['end_date'], "end date somehow initialized ")
        self.assertIsNone(jsondetails['enrollment_start'], "enrollment_start date somehow initialized ")
        self.assertIsNone(jsondetails['enrollment_end'], "enrollment_end date somehow initialized ")
        self.assertIsNone(jsondetails['syllabus'], "syllabus somehow initialized")
        self.assertIsNone(jsondetails['intro_video'], "intro_video somehow initialized")
        self.assertIsNone(jsondetails['effort'], "effort somehow initialized")

    def test_ooc_encoder(self):
        """
        Test the encoder out of its original constrained purpose to see if it functions for general use
        """
        details = {
            'number': 1,
            'string': 'string',
            'datetime': datetime.datetime.now(UTC())
        }
        jsondetails = json.dumps(details, cls=CourseSettingsEncoder)
        jsondetails = json.loads(jsondetails)

        self.assertEquals(1, jsondetails['number'])
        self.assertEqual(jsondetails['string'], 'string')

    def test_update_and_fetch(self):
        jsondetails = CourseDetails.fetch(self.course.id)
        jsondetails.syllabus = "<a href='foo'>bar</a>"
        # encode - decode to convert date fields and other data which changes form
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).syllabus,
            jsondetails.syllabus, "After set syllabus"
        )
        jsondetails.short_description = "Short Description"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).short_description,
            jsondetails.short_description, "After set short_description"
        )
        jsondetails.overview = "Overview"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).overview,
            jsondetails.overview, "After set overview"
        )
        jsondetails.intro_video = "intro_video"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).intro_video,
            jsondetails.intro_video, "After set intro_video"
        )
        jsondetails.effort = "effort"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).effort,
            jsondetails.effort, "After set effort"
        )
        jsondetails.start_date = datetime.datetime(2010, 10, 1, 0, tzinfo=UTC())
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).start_date,
            jsondetails.start_date
        )
        jsondetails.course_image_name = "an_image.jpg"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).course_image_name,
            jsondetails.course_image_name
        )

    @override_settings(MKTG_URLS={'ROOT': 'dummy-root'})
    def test_marketing_site_fetch(self):
        settings_details_url = get_url(self.course.id)

        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            response = self.client.get_html(settings_details_url)
            self.assertNotContains(response, "Course Summary Page")
            self.assertNotContains(response, "Send a note to students via email")
            self.assertContains(response, "course summary page will not be viewable")

            self.assertContains(response, "Course Start Date")
            self.assertContains(response, "Course End Date")
            self.assertContains(response, "Enrollment Start Date")
            self.assertContains(response, "Enrollment End Date")
            self.assertContains(response, "not the dates shown on your course summary page")

            self.assertContains(response, "Introducing Your Course")
            self.assertContains(response, "Course Image")
            self.assertContains(response, "Course Short Description")
            self.assertNotContains(response, "Course Overview")
            self.assertNotContains(response, "Course Introduction Video")
            self.assertNotContains(response, "Requirements")

    def test_editable_short_description_fetch(self):
        settings_details_url = get_url(self.course.id)

        with mock.patch.dict('django.conf.settings.FEATURES', {'EDITABLE_SHORT_DESCRIPTION': False}):
            response = self.client.get_html(settings_details_url)
            self.assertNotContains(response, "Course Short Description")


    def test_regular_site_fetch(self):
        settings_details_url = get_url(self.course.id)

        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': False}):
            response = self.client.get_html(settings_details_url)
            self.assertContains(response, "Course Summary Page")
            self.assertContains(response, "Send a note to students via email")
            self.assertNotContains(response, "course summary page will not be viewable")

            self.assertContains(response, "Course Start Date")
            self.assertContains(response, "Course End Date")
            self.assertContains(response, "Enrollment Start Date")
            self.assertContains(response, "Enrollment End Date")
            self.assertNotContains(response, "not the dates shown on your course summary page")

            self.assertContains(response, "Introducing Your Course")
            self.assertContains(response, "Course Image")
            self.assertContains(response, "Course Short Description")
            self.assertContains(response, "Course Overview")
            self.assertContains(response, "Course Introduction Video")
            self.assertContains(response, "Requirements")


class CourseDetailsViewTest(CourseTestCase):
    """
    Tests for modifying content on the first course settings page (course dates, overview, etc.).
    """
    def alter_field(self, url, details, field, val):
        """
        Change the one field to the given value and then invoke the update post to see if it worked.
        """
        setattr(details, field, val)
        # Need to partially serialize payload b/c the mock doesn't handle it correctly
        payload = copy.copy(details.__dict__)
        payload['start_date'] = CourseDetailsViewTest.convert_datetime_to_iso(details.start_date)
        payload['end_date'] = CourseDetailsViewTest.convert_datetime_to_iso(details.end_date)
        payload['enrollment_start'] = CourseDetailsViewTest.convert_datetime_to_iso(details.enrollment_start)
        payload['enrollment_end'] = CourseDetailsViewTest.convert_datetime_to_iso(details.enrollment_end)
        resp = self.client.ajax_post(url, payload)
        self.compare_details_with_encoding(json.loads(resp.content), details.__dict__, field + str(val))

    @staticmethod
    def convert_datetime_to_iso(datetime_obj):
        """
        Use the xblock serializer to convert the datetime
        """
        return Date().to_json(datetime_obj)

    def test_update_and_fetch(self):
        details = CourseDetails.fetch(self.course.id)

        # resp s/b json from here on
        url = get_url(self.course.id)
        resp = self.client.get_json(url)
        self.compare_details_with_encoding(json.loads(resp.content), details.__dict__, "virgin get")

        utc = UTC()
        self.alter_field(url, details, 'start_date', datetime.datetime(2012, 11, 12, 1, 30, tzinfo=utc))
        self.alter_field(url, details, 'start_date', datetime.datetime(2012, 11, 1, 13, 30, tzinfo=utc))
        self.alter_field(url, details, 'end_date', datetime.datetime(2013, 2, 12, 1, 30, tzinfo=utc))
        self.alter_field(url, details, 'enrollment_start', datetime.datetime(2012, 10, 12, 1, 30, tzinfo=utc))

        self.alter_field(url, details, 'enrollment_end', datetime.datetime(2012, 11, 15, 1, 30, tzinfo=utc))
        self.alter_field(url, details, 'short_description', "Short Description")
        self.alter_field(url, details, 'overview', "Overview")
        self.alter_field(url, details, 'intro_video', "intro_video")
        self.alter_field(url, details, 'effort', "effort")
        self.alter_field(url, details, 'course_image_name', "course_image_name")

    def compare_details_with_encoding(self, encoded, details, context):
        """
        compare all of the fields of the before and after dicts
        """
        self.compare_date_fields(details, encoded, context, 'start_date')
        self.compare_date_fields(details, encoded, context, 'end_date')
        self.compare_date_fields(details, encoded, context, 'enrollment_start')
        self.compare_date_fields(details, encoded, context, 'enrollment_end')
        self.assertEqual(details['short_description'], encoded['short_description'], context + " short_description not ==")
        self.assertEqual(details['overview'], encoded['overview'], context + " overviews not ==")
        self.assertEqual(details['intro_video'], encoded.get('intro_video', None), context + " intro_video not ==")
        self.assertEqual(details['effort'], encoded['effort'], context + " efforts not ==")
        self.assertEqual(details['course_image_name'], encoded['course_image_name'], context + " images not ==")

    def compare_date_fields(self, details, encoded, context, field):
        """
        Compare the given date fields between the before and after doing json deserialization
        """
        if details[field] is not None:
            date = Date()
            if field in encoded and encoded[field] is not None:
                dt1 = date.from_json(encoded[field])
                dt2 = details[field]

                self.assertEqual(dt1, dt2, msg="{} != {} at {}".format(dt1, dt2, context))
            else:
                self.fail(field + " missing from encoded but in details at " + context)
        elif field in encoded and encoded[field] is not None:
            self.fail(field + " included in encoding but missing from details at " + context)


class CourseGradingTest(CourseTestCase):
    """
    Tests for the course settings grading page.
    """
    def test_initial_grader(self):
        test_grader = CourseGradingModel(self.course)
        self.assertIsNotNone(test_grader.graders)
        self.assertIsNotNone(test_grader.grade_cutoffs)

    def test_fetch_grader(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        self.assertIsNotNone(test_grader.graders, "No graders")
        self.assertIsNotNone(test_grader.grade_cutoffs, "No cutoffs")

        for i, grader in enumerate(test_grader.graders):
            subgrader = CourseGradingModel.fetch_grader(self.course.id, i)
            self.assertDictEqual(grader, subgrader, str(i) + "th graders not equal")

    def test_update_from_json(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "Noop update")

        test_grader.graders[0]['weight'] = test_grader.graders[0].get('weight') * 2
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "Weight[0] * 2")

        test_grader.grade_cutoffs['D'] = 0.3
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "cutoff add D")

        test_grader.grace_period = {'hours': 4, 'minutes': 5, 'seconds': 0}
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "4 hour grace period")

    def test_update_grader_from_json(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        altered_grader = CourseGradingModel.update_grader_from_json(
            self.course.id, test_grader.graders[1], self.user
        )
        self.assertDictEqual(test_grader.graders[1], altered_grader, "Noop update")

        test_grader.graders[1]['min_count'] = test_grader.graders[1].get('min_count') + 2
        altered_grader = CourseGradingModel.update_grader_from_json(
            self.course.id, test_grader.graders[1], self.user)
        self.assertDictEqual(test_grader.graders[1], altered_grader, "min_count[1] + 2")

        test_grader.graders[1]['drop_count'] = test_grader.graders[1].get('drop_count') + 1
        altered_grader = CourseGradingModel.update_grader_from_json(
            self.course.id, test_grader.graders[1], self.user)
        self.assertDictEqual(test_grader.graders[1], altered_grader, "drop_count[1] + 2")

    def test_update_cutoffs_from_json(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user)
        # Unlike other tests, need to actually perform a db fetch for this test since update_cutoffs_from_json
        #  simply returns the cutoffs you send into it, rather than returning the db contents.
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "Noop update")

        test_grader.grade_cutoffs['D'] = 0.3
        CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user)
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "cutoff add D")

        test_grader.grade_cutoffs['Pass'] = 0.75
        CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user)
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "cutoff change 'Pass'")

    def test_delete_grace_period(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        CourseGradingModel.update_grace_period_from_json(
            self.course.id, test_grader.grace_period, self.user
        )
        # update_grace_period_from_json doesn't return anything, so query the db for its contents.
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertEqual(test_grader.grace_period, altered_grader.grace_period, "Noop update")

        test_grader.grace_period = {'hours': 15, 'minutes': 5, 'seconds': 30}
        CourseGradingModel.update_grace_period_from_json(
            self.course.id, test_grader.grace_period, self.user)
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grace_period, altered_grader.grace_period, "Adding in a grace period")

        test_grader.grace_period = {'hours': 1, 'minutes': 10, 'seconds': 0}
        # Now delete the grace period
        CourseGradingModel.delete_grace_period(self.course.id, self.user)
        # update_grace_period_from_json doesn't return anything, so query the db for its contents.
        altered_grader = CourseGradingModel.fetch(self.course.id)
        # Once deleted, the grace period should simply be None
        self.assertEqual(None, altered_grader.grace_period, "Delete grace period")

    def test_update_section_grader_type(self):
        # Get the descriptor and the section_grader_type and assert they are the default values
        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
        section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)

        self.assertEqual('notgraded', section_grader_type['graderType'])
        self.assertEqual(None, descriptor.format)
        self.assertEqual(False, descriptor.graded)

        # Change the default grader type to Homework, which should also mark the section as graded
        CourseGradingModel.update_section_grader_type(self.course, 'Homework', self.user)
        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
        section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)

        self.assertEqual('Homework', section_grader_type['graderType'])
        self.assertEqual('Homework', descriptor.format)
        self.assertEqual(True, descriptor.graded)

        # Change the grader type back to notgraded, which should also unmark the section as graded
        CourseGradingModel.update_section_grader_type(self.course, 'notgraded', self.user)
        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
        section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)

        self.assertEqual('notgraded', section_grader_type['graderType'])
        self.assertEqual(None, descriptor.format)
        self.assertEqual(False, descriptor.graded)

    def test_get_set_grader_types_ajax(self):
        """
        Test configuring the graders via ajax calls
        """
        grader_type_url_base = get_url(self.course.id, 'grading_handler')
        # test get whole
        response = self.client.get_json(grader_type_url_base)
        whole_model = json.loads(response.content)
        self.assertIn('graders', whole_model)
        self.assertIn('grade_cutoffs', whole_model)
        self.assertIn('grace_period', whole_model)
        # test post/update whole
        whole_model['grace_period'] = {'hours': 1, 'minutes': 30, 'seconds': 0}
        response = self.client.ajax_post(grader_type_url_base, whole_model)
        self.assertEqual(200, response.status_code)
        response = self.client.get_json(grader_type_url_base)
        whole_model = json.loads(response.content)
        self.assertEqual(whole_model['grace_period'], {'hours': 1, 'minutes': 30, 'seconds': 0})
        # test get one grader
        self.assertGreater(len(whole_model['graders']), 1)  # ensure test will make sense
        response = self.client.get_json(grader_type_url_base + '/1')
        grader_sample = json.loads(response.content)
        self.assertEqual(grader_sample, whole_model['graders'][1])
        # test add grader
        new_grader = {
            "type": "Extra Credit",
            "min_count": 1,
            "drop_count": 2,
            "short_label": None,
            "weight": 15,
        }
        response = self.client.ajax_post(
            '{}/{}'.format(grader_type_url_base, len(whole_model['graders'])),
            new_grader
        )
        self.assertEqual(200, response.status_code)
        grader_sample = json.loads(response.content)
        new_grader['id'] = len(whole_model['graders'])
        self.assertEqual(new_grader, grader_sample)
        # test delete grader
        response = self.client.delete(grader_type_url_base + '/1', HTTP_ACCEPT="application/json")
        self.assertEqual(204, response.status_code)
        response = self.client.get_json(grader_type_url_base)
        updated_model = json.loads(response.content)
        new_grader['id'] -= 1  # one fewer and the id mutates
        self.assertIn(new_grader, updated_model['graders'])
        self.assertNotIn(whole_model['graders'][1], updated_model['graders'])

    def setup_test_set_get_section_grader_ajax(self):
        """
        Populate the course, grab a section, get the url for the assignment type access
        """
        self.populate_course()
        sequential_usage_key = self.course.id.make_usage_key("sequential", None)
        sections = get_modulestore(self.course.id).get_items(sequential_usage_key)
        # see if test makes sense
        self.assertGreater(len(sections), 0, "No sections found")
        section = sections[0]  # just take the first one
        return reverse_usage_url('xblock_handler', section.location)

    def test_set_get_section_grader_ajax(self):
        """
        Test setting and getting section grades via the grade as url
        """
        grade_type_url = self.setup_test_set_get_section_grader_ajax()
        response = self.client.ajax_post(grade_type_url, {'graderType': u'Homework'})
        self.assertEqual(200, response.status_code)
        response = self.client.get_json(grade_type_url + '?fields=graderType')
        self.assertEqual(json.loads(response.content).get('graderType'), u'Homework')
        # and unset
        response = self.client.ajax_post(grade_type_url, {'graderType': u'notgraded'})
        self.assertEqual(200, response.status_code)
        response = self.client.get_json(grade_type_url + '?fields=graderType')
        self.assertEqual(json.loads(response.content).get('graderType'), u'notgraded')


class CourseMetadataEditingTest(CourseTestCase):
    """
    Tests for CourseMetadata.
    """
    def setUp(self):
        CourseTestCase.setUp(self)
        self.fullcourse = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
        self.course_setting_url = get_url(self.course.id, 'advanced_settings_handler')
        self.fullcourse_setting_url = get_url(self.fullcourse.id, 'advanced_settings_handler')

    def test_fetch_initial_fields(self):
        test_model = CourseMetadata.fetch(self.course)
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")

        test_model = CourseMetadata.fetch(self.fullcourse)
        self.assertNotIn('graceperiod', test_model, 'blacklisted field leaked in')
        self.assertIn('display_name', test_model, 'full missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('rerandomize', test_model, 'Missing rerandomize metadata field')
        self.assertIn('showanswer', test_model, 'showanswer field ')
        self.assertIn('xqa_key', test_model, 'xqa_key field ')

    def test_update_from_json(self):
        test_model = CourseMetadata.update_from_json(
            self.course,
            {
                "advertised_start": "start A",
                "days_early_for_beta": 2,
            },
            user=self.user
         )
        self.update_check(test_model)
        # try fresh fetch to ensure persistence
        fresh = modulestore('direct').get_course(self.course.id)
        test_model = CourseMetadata.fetch(fresh)
        self.update_check(test_model)
        # now change some of the existing metadata
        test_model = CourseMetadata.update_from_json(
            fresh,
            {
                "advertised_start": "start B",
                "display_name": "jolly roger",
            },
            user=self.user
        )
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'jolly roger', "not expected value")
        self.assertIn('advertised_start', test_model, 'Missing revised advertised_start metadata field')
        self.assertEqual(test_model['advertised_start'], 'start B', "advertised_start not expected value")

    def update_check(self, test_model):
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('advertised_start', test_model, 'Missing new advertised_start metadata field')
        self.assertEqual(test_model['advertised_start'], 'start A', "advertised_start not expected value")
        self.assertIn('days_early_for_beta', test_model, 'Missing days_early_for_beta metadata field')
        self.assertEqual(test_model['days_early_for_beta'], 2, "days_early_for_beta not expected value")

    def test_delete_key(self):
        test_model = CourseMetadata.update_from_json(
            self.fullcourse,
            {"unsetKeys": ['showanswer', 'xqa_key']},
            user=self.user
        )
        # ensure no harm
        self.assertNotIn('graceperiod', test_model, 'blacklisted field leaked in')
        self.assertIn('display_name', test_model, 'full missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('rerandomize', test_model, 'Missing rerandomize metadata field')
        # check for deletion effectiveness
        self.assertEqual('finished', test_model['showanswer'], 'showanswer field still in')
        self.assertEqual(None, test_model['xqa_key'], 'xqa_key field still in')

    def test_http_fetch_initial_fields(self):
        response = self.client.get_json(self.course_setting_url)
        test_model = json.loads(response.content)
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")

        response = self.client.get_json(self.fullcourse_setting_url)
        test_model = json.loads(response.content)
        self.assertNotIn('graceperiod', test_model, 'blacklisted field leaked in')
        self.assertIn('display_name', test_model, 'full missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('rerandomize', test_model, 'Missing rerandomize metadata field')
        self.assertIn('showanswer', test_model, 'showanswer field ')
        self.assertIn('xqa_key', test_model, 'xqa_key field ')

    def test_http_update_from_json(self):
        response = self.client.ajax_post(self.course_setting_url, {
            "advertised_start": "start A",
            "testcenter_info": {"c": "test"},
            "days_early_for_beta": 2,
            "unsetKeys": ['showanswer', 'xqa_key'],
        })
        test_model = json.loads(response.content)
        self.update_check(test_model)
        self.assertEqual('finished', test_model['showanswer'], 'showanswer field still in')
        self.assertEqual(None, test_model['xqa_key'], 'xqa_key field still in')

        response = self.client.get_json(self.course_setting_url)
        test_model = json.loads(response.content)
        self.update_check(test_model)
        # now change some of the existing metadata
        response = self.client.ajax_post(self.course_setting_url, {
            "advertised_start": "start B",
            "display_name": "jolly roger"
        })
        test_model = json.loads(response.content)
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'jolly roger', "not expected value")
        self.assertIn('advertised_start', test_model, 'Missing revised advertised_start metadata field')
        self.assertEqual(test_model['advertised_start'], 'start B', "advertised_start not expected value")

    def test_advanced_components_munge_tabs(self):
        """
        Test that adding and removing specific advanced components adds and removes tabs.
        """
        self.assertNotIn(EXTRA_TAB_PANELS.get("open_ended"), self.course.tabs)
        self.assertNotIn(EXTRA_TAB_PANELS.get("notes"), self.course.tabs)
        self.client.ajax_post(self.course_setting_url, {
            ADVANCED_COMPONENT_POLICY_KEY: ["combinedopenended"]
        })
        course = modulestore().get_course(self.course.id)
        self.assertIn(EXTRA_TAB_PANELS.get("open_ended"), course.tabs)
        self.assertNotIn(EXTRA_TAB_PANELS.get("notes"), course.tabs)
        self.client.ajax_post(self.course_setting_url, {
            ADVANCED_COMPONENT_POLICY_KEY: []
        })
        course = modulestore().get_course(self.course.id)
        self.assertNotIn(EXTRA_TAB_PANELS.get("open_ended"), course.tabs)


class CourseGraderUpdatesTest(CourseTestCase):
    """
    Test getting, deleting, adding, & updating graders
    """
    def setUp(self):
        """Compute the url to use in tests"""
        super(CourseGraderUpdatesTest, self).setUp()
        self.url = get_url(self.course.id, 'grading_handler')
        self.starting_graders = CourseGradingModel(self.course).graders

    def test_get(self):
        """Test getting a specific grading type record."""
        resp = self.client.get_json(self.url + '/0')
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(self.starting_graders[0], obj)

    def test_delete(self):
        """Test deleting a specific grading type record."""
        resp = self.client.delete(self.url + '/0', HTTP_ACCEPT="application/json")
        self.assertEqual(resp.status_code, 204)
        current_graders = CourseGradingModel.fetch(self.course.id).graders
        self.assertNotIn(self.starting_graders[0], current_graders)
        self.assertEqual(len(self.starting_graders) - 1, len(current_graders))

    def test_update(self):
        """Test updating a specific grading type record."""
        grader = {
            "id": 0,
            "type": "manual",
            "min_count": 5,
            "drop_count": 10,
            "short_label": "yo momma",
            "weight": 17.3,
        }
        resp = self.client.ajax_post(self.url + '/0', grader)
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(obj, grader)
        current_graders = CourseGradingModel.fetch(self.course.id).graders
        self.assertEqual(len(self.starting_graders), len(current_graders))

    def test_add(self):
        """Test adding a grading type record."""
        # the same url works for changing the whole grading model (graceperiod, cutoffs, and grading types) when
        # the grading_index is None; thus, using None to imply adding a grading_type doesn't work; so, it uses an
        # index out of bounds to imply create item.
        grader = {
            "type": "manual",
            "min_count": 5,
            "drop_count": 10,
            "short_label": "yo momma",
            "weight": 17.3,
        }
        resp = self.client.ajax_post('{}/{}'.format(self.url, len(self.starting_graders) + 1), grader)
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(obj['id'], len(self.starting_graders))
        del obj['id']
        self.assertEqual(obj, grader)
        current_graders = CourseGradingModel.fetch(self.course.id).graders
        self.assertEqual(len(self.starting_graders) + 1, len(current_graders))

########NEW FILE########
__FILENAME__ = test_crud
import unittest

from xmodule import templates
from xmodule.modulestore.tests import persistent_factories
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.django import modulestore, loc_mapper, clear_existing_modulestores
from xmodule.seq_module import SequenceDescriptor
from xmodule.capa_module import CapaDescriptor
from xmodule.modulestore.locator import BlockUsageLocator, LocalId
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateCourseError
from xmodule.html_module import HtmlDescriptor
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase


class TemplateTests(unittest.TestCase):
    """
    Test finding and using the templates (boilerplates) for xblocks.
    """

    def setUp(self):
        clear_existing_modulestores()  # redundant w/ cleanup but someone was getting errors
        self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
        self.addCleanup(clear_existing_modulestores)

    def test_get_templates(self):
        found = templates.all_templates()
        self.assertIsNotNone(found.get('course'))
        self.assertIsNotNone(found.get('about'))
        self.assertIsNotNone(found.get('html'))
        self.assertIsNotNone(found.get('problem'))
        self.assertEqual(len(found.get('course')), 0)
        self.assertEqual(len(found.get('about')), 1)
        self.assertGreaterEqual(len(found.get('html')), 2)
        self.assertGreaterEqual(len(found.get('problem')), 10)
        dropdown = None
        for template in found['problem']:
            self.assertIn('metadata', template)
            self.assertIn('display_name', template['metadata'])
            if template['metadata']['display_name'] == 'Dropdown':
                dropdown = template
                break
        self.assertIsNotNone(dropdown)
        self.assertIn('markdown', dropdown['metadata'])
        self.assertIn('data', dropdown)
        self.assertRegexpMatches(dropdown['metadata']['markdown'], r'^Dropdown.*')
        self.assertRegexpMatches(dropdown['data'], r'<problem>\s*<p>Dropdown.*')

    def test_get_some_templates(self):
        self.assertEqual(len(SequenceDescriptor.templates()), 0)
        self.assertGreater(len(HtmlDescriptor.templates()), 0)
        self.assertIsNone(SequenceDescriptor.get_template('doesntexist.yaml'))
        self.assertIsNone(HtmlDescriptor.get_template('doesntexist.yaml'))
        self.assertIsNotNone(HtmlDescriptor.get_template('announcement.yaml'))

    def test_factories(self):
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='tempcourse', org='testx',
            display_name='fun test course', user_id='testbot'
        )
        self.assertIsInstance(test_course, CourseDescriptor)
        self.assertEqual(test_course.display_name, 'fun test course')
        index_info = modulestore('split').get_course_index_info(test_course.id)
        self.assertEqual(index_info['org'], 'testx')
        self.assertEqual(index_info['offering'], 'tempcourse')

        test_chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
            parent_location=test_course.location)
        self.assertIsInstance(test_chapter, SequenceDescriptor)
        # refetch parent which should now point to child
        test_course = modulestore('split').get_course(test_course.id.version_agnostic())
        self.assertIn(test_chapter.location, test_course.children)

        with self.assertRaises(DuplicateCourseError):
            persistent_factories.PersistentCourseFactory.create(
                offering='tempcourse', org='testx',
                display_name='fun test course', user_id='testbot'
            )

    def test_temporary_xblocks(self):
        """
        Test create_xblock to create non persisted xblocks
        """
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='tempcourse', org='testx',
            display_name='fun test course', user_id='testbot'
        )

        test_chapter = modulestore('split').create_xblock(
            test_course.system, 'chapter', {'display_name': 'chapter n'}, parent_xblock=test_course
        )
        self.assertIsInstance(test_chapter, SequenceDescriptor)
        self.assertEqual(test_chapter.display_name, 'chapter n')
        self.assertIn(test_chapter, test_course.get_children())

        # test w/ a definition (e.g., a problem)
        test_def_content = '<problem>boo</problem>'
        test_problem = modulestore('split').create_xblock(
            test_course.system, 'problem', {'data': test_def_content}, parent_xblock=test_chapter
        )
        self.assertIsInstance(test_problem, CapaDescriptor)
        self.assertEqual(test_problem.data, test_def_content)
        self.assertIn(test_problem, test_chapter.get_children())
        test_problem.display_name = 'test problem'
        self.assertEqual(test_problem.display_name, 'test problem')

    def test_persist_dag(self):
        """
        try saving temporary xblocks
        """
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='tempcourse', org='testx',
            display_name='fun test course', user_id='testbot'
        )
        test_chapter = modulestore('split').create_xblock(
            test_course.system, 'chapter', {'display_name': 'chapter n'}, parent_xblock=test_course
        )
        self.assertEqual(test_chapter.display_name, 'chapter n')
        test_def_content = '<problem>boo</problem>'
        # create child
        new_block = modulestore('split').create_xblock(
            test_course.system,
            'problem',
            fields={
                'data': test_def_content,
                'display_name': 'problem'
            },
            parent_xblock=test_chapter
        )
        self.assertIsNotNone(new_block.definition_locator)
        self.assertTrue(isinstance(new_block.definition_locator.definition_id, LocalId))
        # better to pass in persisted parent over the subdag so
        # subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
        # persist parent
        persisted_course = modulestore('split').persist_xblock_dag(test_course, 'testbot')
        self.assertEqual(len(persisted_course.children), 1)
        persisted_chapter = persisted_course.get_children()[0]
        self.assertEqual(persisted_chapter.category, 'chapter')
        self.assertEqual(persisted_chapter.display_name, 'chapter n')
        self.assertEqual(len(persisted_chapter.children), 1)
        persisted_problem = persisted_chapter.get_children()[0]
        self.assertEqual(persisted_problem.category, 'problem')
        self.assertEqual(persisted_problem.data, test_def_content)
        # update it
        persisted_problem.display_name = 'altered problem'
        persisted_problem = modulestore('split').persist_xblock_dag(persisted_problem, 'testbot')
        self.assertEqual(persisted_problem.display_name, 'altered problem')

    def test_delete_course(self):
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='history.doomed', org='edu.harvard',
            display_name='doomed test course',
            user_id='testbot')
        persistent_factories.ItemFactory.create(display_name='chapter 1',
            parent_location=test_course.location)

        id_locator = test_course.id.for_branch('draft')
        guid_locator = test_course.location.course_agnostic()
        # verify it can be retrieved by id
        self.assertIsInstance(modulestore('split').get_course(id_locator), CourseDescriptor)
        # and by guid
        self.assertIsInstance(modulestore('split').get_item(guid_locator), CourseDescriptor)
        modulestore('split').delete_course(id_locator)
        # test can no longer retrieve by id
        self.assertRaises(ItemNotFoundError, modulestore('split').get_course, id_locator)
        # but can by guid
        self.assertIsInstance(modulestore('split').get_item(guid_locator), CourseDescriptor)

    def test_block_generations(self):
        """
        Test get_block_generations
        """
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='history.hist101', org='edu.harvard',
            display_name='history test course',
            user_id='testbot'
        )
        chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
            parent_location=test_course.location, user_id='testbot')
        sub = persistent_factories.ItemFactory.create(display_name='subsection 1',
            parent_location=chapter.location, user_id='testbot', category='vertical')
        first_problem = persistent_factories.ItemFactory.create(
            display_name='problem 1', parent_location=sub.location, user_id='testbot', category='problem',
            data="<problem></problem>"
        )
        first_problem.max_attempts = 3
        first_problem.save()  # decache the above into the kvs
        updated_problem = modulestore('split').update_item(first_problem, '**replace_user**')
        self.assertIsNotNone(updated_problem.previous_version)
        self.assertEqual(updated_problem.previous_version, first_problem.update_version)
        self.assertNotEqual(updated_problem.update_version, first_problem.update_version)
        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot', delete_children=True)

        second_problem = persistent_factories.ItemFactory.create(
            display_name='problem 2',
            parent_location=BlockUsageLocator.make_relative(
                updated_loc, block_type='problem', block_id=sub.location.block_id
            ),
            user_id='testbot', category='problem',
            data="<problem></problem>"
        )

        # course root only updated 2x
        version_history = modulestore('split').get_block_generations(test_course.location)
        self.assertEqual(version_history.locator.version_guid, test_course.location.version_guid)
        self.assertEqual(len(version_history.children), 1)
        self.assertEqual(version_history.children[0].children, [])
        self.assertEqual(version_history.children[0].locator.version_guid, chapter.location.version_guid)

        # sub changed on add, add problem, delete problem, add problem in strict linear seq
        version_history = modulestore('split').get_block_generations(sub.location)
        self.assertEqual(len(version_history.children), 1)
        self.assertEqual(len(version_history.children[0].children), 1)
        self.assertEqual(len(version_history.children[0].children[0].children), 1)
        self.assertEqual(len(version_history.children[0].children[0].children[0].children), 0)

        # first and second problem may show as same usage_id; so, need to ensure their histories are right
        version_history = modulestore('split').get_block_generations(updated_problem.location)
        self.assertEqual(version_history.locator.version_guid, first_problem.location.version_guid)
        self.assertEqual(len(version_history.children), 1)  # updated max_attempts
        self.assertEqual(len(version_history.children[0].children), 0)

        version_history = modulestore('split').get_block_generations(second_problem.location)
        self.assertNotEqual(version_history.locator.version_guid, first_problem.location.version_guid)

    def test_split_inject_loc_mapper(self):
        """
        Test that creating a loc_mapper causes it to automatically attach to the split mongo store
        """
        # instantiate location mapper before split
        mapper = loc_mapper()
        # split must inject the location mapper itself since the mapper existed before it did
        self.assertEqual(modulestore('split').loc_mapper, mapper)

    def test_loc_inject_into_split(self):
        """
        Test that creating a loc_mapper causes it to automatically attach to the split mongo store
        """
        # force instantiation of split modulestore before there's a location mapper and verify
        # it has no pointer to loc mapper
        self.assertIsNone(modulestore('split').loc_mapper)
        # force instantiation of location mapper which must inject itself into the split
        mapper = loc_mapper()
        self.assertEqual(modulestore('split').loc_mapper, mapper)


########NEW FILE########
__FILENAME__ = test_export_git
"""
Test the ability to export courses to xml from studio
"""

import copy
import os
import shutil
import subprocess
from uuid import uuid4

from django.conf import settings
from django.test.utils import override_settings
from pymongo import MongoClient

from .utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestExportGit(CourseTestCase):
    """
    Tests pushing a course to a git repository
    """

    def setUp(self):
        """
        Setup test course, user, and url.
        """
        super(TestExportGit, self).setUp()
        self.course_module = modulestore().get_course(self.course.id)
        self.test_url = reverse_course_url('export_git', self.course.id)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_giturl_missing(self):
        """
        Test to make sure an appropriate error is displayed
        if course hasn't set giturl.
        """
        response = self.client.get(self.test_url)
        self.assertEqual(200, response.status_code)
        self.assertIn(
            ('giturl must be defined in your '
             'course settings before you can export to git.'),
            response.content
        )

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertEqual(200, response.status_code)
        self.assertIn(
            ('giturl must be defined in your '
             'course settings before you can export to git.'),
            response.content
        )

    def test_course_export_failures(self):
        """
        Test failed course export response.
        """
        self.course_module.giturl = 'foobar'
        get_modulestore(self.course_module.location).update_item(self.course_module)

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertIn('Export Failed:', response.content)

    def test_exception_translation(self):
        """
        Regression test for making sure errors are properly stringified
        """
        self.course_module.giturl = 'foobar'
        get_modulestore(self.course_module.location).update_item(self.course_module)

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertNotIn('django.utils.functional.__proxy__', response.content)

    def test_course_export_success(self):
        """
        Test successful course export response.
        """
        # Build out local bare repo, and set course git url to it
        repo_dir = os.path.abspath(git_export_utils.GIT_REPO_EXPORT_DIR)
        os.mkdir(repo_dir)
        self.addCleanup(shutil.rmtree, repo_dir)

        bare_repo_dir = '{0}/test_repo.git'.format(
            os.path.abspath(git_export_utils.GIT_REPO_EXPORT_DIR))
        os.mkdir(bare_repo_dir)
        self.addCleanup(shutil.rmtree, bare_repo_dir)

        subprocess.check_output(['git', '--bare', 'init', ], cwd=bare_repo_dir)

        self.populate_course()
        self.course_module.giturl = 'file://{}'.format(bare_repo_dir)
        get_modulestore(self.course_module.location).update_item(self.course_module)

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertIn('Export Succeeded', response.content)

########NEW FILE########
__FILENAME__ = test_i18n
from unittest import skip

from django.contrib.auth.models import User
from django.test.utils import override_settings

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient


@override_settings(MODULESTORE=TEST_MODULESTORE)
class InternationalizationTest(ModuleStoreTestCase):
    """
    Tests to validate Internationalization.
    """

    def setUp(self):
        """
        These tests need a user in the DB so that the django Test Client
        can log them in.
        They inherit from the ModuleStoreTestCase class so that the mongodb collection
        will be cleared out before each test case execution and deleted
        afterwards.
        """
        self.uname = 'testuser'
        self.email = 'test+courses@edx.org'
        self.password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(self.uname, self.email, self.password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
        }

    def test_course_plain_english(self):
        """Test viewing the index page with no courses"""
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.uname, password=self.password)

        resp = self.client.get_html('/course/')
        self.assertContains(resp,
                            '<h1 class="page-header">My Courses</h1>',
                            status_code=200,
                            html=True)

    def test_course_explicit_english(self):
        """Test viewing the index page with no courses"""
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.uname, password=self.password)

        resp = self.client.get_html('/course/',
                               {},
                               HTTP_ACCEPT_LANGUAGE='en'
                               )

        self.assertContains(resp,
                            '<h1 class="page-header">My Courses</h1>',
                            status_code=200,
                            html=True)

    # ****
    # NOTE:
    # ****
    #
    # This test will break when we replace this fake 'test' language
    # with actual Esperanto. This test will need to be updated with
    # actual Esperanto at that time.
    # Test temporarily disable since it depends on creation of dummy strings
    @skip
    def test_course_with_accents(self):
        """Test viewing the index page with no courses"""
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.uname, password=self.password)

        resp = self.client.get_html(
            '/course/',
            {},
            HTTP_ACCEPT_LANGUAGE='eo'
        )

        TEST_STRING = (
            u'<h1 class="title-1">'
            u'My \xc7\xf6\xfcrs\xe9s L#'
            u'</h1>'
        )

        self.assertContains(resp,
                            TEST_STRING,
                            status_code=200,
                            html=True)

########NEW FILE########
__FILENAME__ = test_import
# -*- coding: utf-8 -*-
# pylint: disable=E1101
"""
Tests for import_from_xml using the mongo modulestore.
"""

from django.test.client import Client
from django.test.utils import override_settings
from django.conf import settings
from path import path
import copy

from django.contrib.auth.models import User

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE

from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import _CONTENTSTORE

from xmodule.exceptions import NotFoundError
from uuid import uuid4
from pymongo import MongoClient

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreImportTest(ModuleStoreTestCase):
    """
    Tests that rely on the toy and test_import_course courses.
    NOTE: refactor using CourseFactory so they do not.
    """
    def setUp(self):

        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True

        # Save the data that we've just changed to the db.
        self.user.save()

        self.client = Client()
        self.client.login(username=uname, password=password)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def load_test_import_course(self):
        '''
        Load the standard course used to test imports
        (for do_import_static=False behavior).
        '''
        content_store = contentstore()
        module_store = modulestore('direct')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['test_import_course'],
            static_content_store=content_store,
            do_import_static=False,
            verbose=True,
        )
        course_id = SlashSeparatedCourseKey('edX', 'test_import_course', '2012_Fall')
        course = module_store.get_course(course_id)
        self.assertIsNotNone(course)

        return module_store, content_store, course

    def test_import_course_into_similar_namespace(self):
        # Checks to make sure that a course with an org/course like
        # edx/course can be imported into a namespace with an org/course
        # like edx/course_name
        module_store, __, course = self.load_test_import_course()
        __, course_items = import_from_xml(
            module_store,
            'common/test/data',
            ['test_import_course_2'],
            target_course_id=course.id,
            verbose=True,
        )
        self.assertEqual(len(course_items), 1)

    def test_unicode_chars_in_course_name_import(self):
        """
        # Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
        """
        module_store = modulestore('direct')
        course_id = SlashSeparatedCourseKey(u'Юникода', u'unicode_course', u'échantillon')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['2014_Uni'],
            target_course_id=course_id
        )

        course = module_store.get_course(course_id)
        self.assertIsNotNone(course)

        # test that course 'display_name' same as imported course 'display_name'
        self.assertEqual(course.display_name, u"Φυσικά το όνομα Unicode")

    def test_static_import(self):
        '''
        Stuff in static_import should always be imported into contentstore
        '''
        _, content_store, course = self.load_test_import_course()

        # make sure we have ONE asset in our contentstore ("should_be_imported.html")
        all_assets, count = content_store.get_all_content_for_course(course.id)
        print "len(all_assets)=%d" % len(all_assets)
        self.assertEqual(len(all_assets), 1)
        self.assertEqual(count, 1)

        content = None
        try:
            location = AssetLocation.from_deprecated_string(
                '/c4x/edX/test_import_course/asset/should_be_imported.html'
            )
            content = content_store.find(location)
        except NotFoundError:
            pass

        self.assertIsNotNone(content)

        # make sure course.static_asset_path is correct
        print "static_asset_path = {0}".format(course.static_asset_path)
        self.assertEqual(course.static_asset_path, 'test_import_course')

    def test_asset_import_nostatic(self):
        '''
        This test validates that an image asset is NOT imported when do_import_static=False
        '''
        content_store = contentstore()

        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)

        course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))

        # make sure we have NO assets in our contentstore
        all_assets, count = content_store.get_all_content_for_course(course.id)
        self.assertEqual(len(all_assets), 0)
        self.assertEqual(count, 0)

    def test_no_static_link_rewrites_on_import(self):
        module_store = modulestore('direct')
        _, courses = import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
        course_key = courses[0].id

        handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts'))
        self.assertIn('/static/', handouts.data)

        handouts = module_store.get_item(course_key.make_usage_key('html', 'toyhtml'))
        self.assertIn('/static/', handouts.data)

    def test_tab_name_imports_correctly(self):
        _module_store, _content_store, course = self.load_test_import_course()
        print "course tabs = {0}".format(course.tabs)
        self.assertEqual(course.tabs[2]['name'], 'Syllabus')

    def test_rewrite_reference_list(self):
        module_store = modulestore('direct')
        target_course_id = SlashSeparatedCourseKey('testX', 'conditional_copy', 'copy_run')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['conditional'],
            target_course_id=target_course_id
        )
        conditional_module = module_store.get_item(
            target_course_id.make_usage_key('conditional', 'condone')
        )
        self.assertIsNotNone(conditional_module)
        different_course_id = SlashSeparatedCourseKey('edX', 'different_course', 'copy_run')
        self.assertListEqual(
            [
                target_course_id.make_usage_key('problem', 'choiceprob'),
                different_course_id.make_usage_key('html', 'for_testing_import_rewrites')
            ],
            conditional_module.sources_list
        )
        self.assertListEqual(
            [
                target_course_id.make_usage_key('html', 'congrats'),
                target_course_id.make_usage_key('html', 'secret_page')
            ],
            conditional_module.show_tag_list
        )

    def test_rewrite_reference(self):
        module_store = modulestore('direct')
        target_course_id = SlashSeparatedCourseKey('testX', 'peergrading_copy', 'copy_run')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['open_ended'],
            target_course_id=target_course_id
        )
        peergrading_module = module_store.get_item(
            target_course_id.make_usage_key('peergrading', 'PeerGradingLinked')
        )
        self.assertIsNotNone(peergrading_module)
        self.assertEqual(
            target_course_id.make_usage_key('combinedopenended', 'SampleQuestion'),
            peergrading_module.link_to_location
        )

    def test_rewrite_reference_value_dict(self):
        module_store = modulestore('direct')
        target_course_id = SlashSeparatedCourseKey('testX', 'split_test_copy', 'copy_run')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['split_test_module'],
            target_course_id=target_course_id
        )
        split_test_module = module_store.get_item(
            target_course_id.make_usage_key('split_test', 'split1')
        )
        self.assertIsNotNone(split_test_module)
        self.assertEqual(
            {
                "0": target_course_id.make_usage_key('vertical', 'sample_0'),
                "2": target_course_id.make_usage_key('vertical', 'sample_2'),
            },
            split_test_module.group_id_to_child,
        )

########NEW FILE########
__FILENAME__ = test_import_draft_order
from django.test.utils import override_settings

from xmodule.modulestore.xml_importer import import_from_xml

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore

from contentstore.tests.modulestore_config import TEST_MODULESTORE


# This test is in the CMS module because the test configuration to use a draft
# modulestore is dependent on django.
@override_settings(MODULESTORE=TEST_MODULESTORE)
class DraftReorderTestCase(ModuleStoreTestCase):

    def test_order(self):
        store = modulestore('direct')
        draft_store = modulestore('default')
        _, course_items = import_from_xml(store, 'common/test/data/', ['import_draft_order'], draft_store=draft_store)
        course_key = course_items[0].id
        sequential = draft_store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
        verticals = sequential.children

        # The order that files are read in from the file system is not guaranteed (cannot rely on
        # alphabetical ordering, for example). Therefore, I have added a lot of variation in filename and desired
        # ordering so that the test reliably failed with the bug, at least on Linux.
        #
        # 'a', 'b', 'c', 'd', and 'z' are all drafts, with 'index_in_children_list' of
        #  2 ,  4 ,  6 ,  5 , and  0  respectively.
        #
        # '5a05be9d59fc4bb79282c94c9e6b88c7' and 'second' are public verticals.
        self.assertEqual(7, len(verticals))
        self.assertEqual(course_key.make_usage_key('vertical', 'z'), verticals[0])
        self.assertEqual(course_key.make_usage_key('vertical', '5a05be9d59fc4bb79282c94c9e6b88c7'), verticals[1])
        self.assertEqual(course_key.make_usage_key('vertical', 'a'), verticals[2])
        self.assertEqual(course_key.make_usage_key('vertical', 'second'), verticals[3])
        self.assertEqual(course_key.make_usage_key('vertical', 'b'), verticals[4])
        self.assertEqual(course_key.make_usage_key('vertical', 'd'), verticals[5])
        self.assertEqual(course_key.make_usage_key('vertical', 'c'), verticals[6])

        # Now also test that the verticals in a second sequential are correct.
        sequential = draft_store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
        verticals = sequential.children
        # 'asecond' and 'zsecond' are drafts with 'index_in_children_list' 0 and 2, respectively.
        # 'secondsubsection' is a public vertical.
        self.assertEqual(3, len(verticals))
        self.assertEqual(course_key.make_usage_key('vertical', 'asecond'), verticals[0])
        self.assertEqual(course_key.make_usage_key('vertical', 'secondsubsection'), verticals[1])
        self.assertEqual(course_key.make_usage_key('vertical', 'zsecond'), verticals[2])

########NEW FILE########
__FILENAME__ = test_import_pure_xblock
"""
Integration tests for importing courses containing pure XBlocks.
"""

from django.test.utils import override_settings

from xblock.core import XBlock
from xblock.fields import String

from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.draft import as_draft
from contentstore.tests.modulestore_config import TEST_MODULESTORE


class StubXBlock(XBlock):
    """
    Stub XBlock to use in tests.

    The default XBlock implementation will load this XBlock
    from XML, using the lowercase version of the class
    as an element name ("stubxblock") and the field names
    as attributes of that element.

    Example:
        <stubxblock test_field="this is only a test" />
    """
    test_field = String(default="default")


@override_settings(MODULESTORE=TEST_MODULESTORE)
class XBlockImportTest(ModuleStoreTestCase):

    def setUp(self):
        self.store = modulestore('direct')
        self.draft_store = modulestore('default')

    @XBlock.register_temp_plugin(StubXBlock)
    def test_import_public(self):
        self._assert_import(
            'pure_xblock_public',
            'set by xml'
        )

    @XBlock.register_temp_plugin(StubXBlock)
    def test_import_draft(self):
        self._assert_import(
            'pure_xblock_draft',
            'set by xml',
            has_draft=True
        )

    def _assert_import(self, course_dir, expected_field_val, has_draft=False):
        """
        Import a course from XML, then verify that the XBlock was loaded
        with the correct field value.

        Args:
            course_dir (str): The name of the course directory (relative to the test data directory)
            expected_xblock_loc (str): The location of the XBlock in the course.
            expected_field_val (str): The expected value of the XBlock's test field.

        Kwargs:
            has_draft (bool): If true, check that a draft of the XBlock exists with
                the expected field value set.

        """
        _, courses = import_from_xml(
            self.store, 'common/test/data', [course_dir],
            draft_store=self.draft_store
        )

        xblock_location = courses[0].id.make_usage_key('stubxblock', 'xblock_test')

        if has_draft:
            xblock_location = as_draft(xblock_location)

        xblock = self.store.get_item(xblock_location)
        self.assertTrue(isinstance(xblock, StubXBlock))
        self.assertEqual(xblock.test_field, expected_field_val)

        if has_draft:
            draft_xblock = self.draft_store.get_item(xblock_location)
            self.assertTrue(isinstance(draft_xblock, StubXBlock))
            self.assertEqual(draft_xblock.test_field, expected_field_val)

########NEW FILE########
__FILENAME__ = test_orphan
"""
Test finding orphans via the view and django config
"""
import json
from contentstore.tests.utils import CourseTestCase
from student.models import CourseEnrollment
from xmodule.modulestore.django import modulestore
from contentstore.utils import reverse_course_url


class TestOrphan(CourseTestCase):
    """
    Test finding orphans via view and django config
    """
    def setUp(self):
        super(TestOrphan, self).setUp()

        runtime = self.course.runtime

        self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', self.course.location.name, runtime)
        self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', self.course.location.name, runtime)
        self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None, runtime)
        self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
        self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None, runtime)
        self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
        self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None, runtime)
        self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
        self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
        self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)

        self.orphan_url = reverse_course_url('orphan_handler', self.course.id)

    def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
        location = self.course.location.replace(category=category, name=name)
        store = modulestore('direct')
        store.create_and_save_xmodule(location, data, metadata, runtime)
        if parent_name:
            # add child to parent in mongo
            parent_location = self.course.location.replace(category=parent_category, name=parent_name)
            parent = store.get_item(parent_location)
            parent.children.append(location)
            store.update_item(parent, self.user.id)

    def test_mongo_orphan(self):
        """
        Test that old mongo finds the orphans
        """
        orphans = json.loads(
            self.client.get(
                self.orphan_url,
                HTTP_ACCEPT='application/json'
            ).content
        )
        self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
        location = self.course.location.replace(category='chapter', name='OrphanChapter')
        self.assertIn(location.to_deprecated_string(), orphans)
        location = self.course.location.replace(category='vertical', name='OrphanVert')
        self.assertIn(location.to_deprecated_string(), orphans)
        location = self.course.location.replace(category='html', name='OrphanHtml')
        self.assertIn(location.to_deprecated_string(), orphans)

    def test_mongo_orphan_delete(self):
        """
        Test that old mongo deletes the orphans
        """
        self.client.delete(self.orphan_url)
        orphans = json.loads(
            self.client.get(self.orphan_url, HTTP_ACCEPT='application/json').content
        )
        self.assertEqual(len(orphans), 0, "Orphans not deleted {}".format(orphans))

    def test_not_permitted(self):
        """
        Test that auth restricts get and delete appropriately
        """
        test_user_client, test_user = self.create_non_staff_authed_user_client()
        CourseEnrollment.enroll(test_user, self.course.id)
        response = test_user_client.get(self.orphan_url)
        self.assertEqual(response.status_code, 403)
        response = test_user_client.delete(self.orphan_url)
        self.assertEqual(response.status_code, 403)

########NEW FILE########
__FILENAME__ = test_permissions
"""
Test CRUD for authorization.
"""
import copy

from django.test.utils import override_settings
from django.contrib.auth.models import User

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_url, reverse_course_url
from student.roles import CourseInstructorRole, CourseStaffRole
from contentstore.views.access import has_course_access
from student import auth


@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestCourseAccess(ModuleStoreTestCase):
    """
    Course-based access (as opposed to access of a non-course xblock)
    """
    def setUp(self):
        """
        Create a staff user and log them in (creating the client).

        Create a pool of users w/o granting them any permissions
        """
        super(TestCourseAccess, self).setUp()
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

        # create a course via the view handler which has a different strategy for permissions than the factory
        self.course_key = SlashSeparatedCourseKey('myu', 'mydept.mycourse', 'myrun')
        course_url = reverse_url('course_handler')
        self.client.ajax_post(course_url,
            {
                'org': self.course_key.org,
                'number': self.course_key.course,
                'display_name': 'My favorite course',
                'run': self.course_key.run,
            }
        )

        self.users = self._create_users()

    def _create_users(self):
        """
        Create 8 users and return them
        """
        users = []
        for i in range(8):
            username = "user{}".format(i)
            email = "test+user{}@edx.org".format(i)
            user = User.objects.create_user(username, email, 'foo')
            user.is_active = True
            user.save()
            users.append(user)
        return users

    def tearDown(self):
        """
        Reverse the setup
        """
        self.client.logout()
        ModuleStoreTestCase.tearDown(self)

    def test_get_all_users(self):
        """
        Test getting all authors for a course where their permissions run the gamut of allowed group
        types.
        """
        # first check the course creator.has explicit access (don't use has_access as is_staff
        # will trump the actual test)
        self.assertTrue(
            CourseInstructorRole(self.course_key).has_user(self.user),
            "Didn't add creator as instructor."
        )
        users = copy.copy(self.users)
        # doesn't use role.users_with_role b/c it's verifying the roles.py behavior
        user_by_role = {}
        # add the misc users to the course in different groups
        for role in [CourseInstructorRole, CourseStaffRole]:
            user_by_role[role] = []
            # pylint: disable=protected-access
            group = role(self.course_key)
            # NOTE: this loop breaks the roles.py abstraction by purposely assigning
            # users to one of each possible groupname in order to test that has_course_access
            # and remove_user work
            user = users.pop()
            group.add_users(user)
            user_by_role[role].append(user)
            self.assertTrue(has_course_access(user, self.course_key), "{} does not have access".format(user))

        course_team_url = reverse_course_url('course_team_handler', self.course_key)
        response = self.client.get_html(course_team_url)
        for role in [CourseInstructorRole, CourseStaffRole]:
            for user in user_by_role[role]:
                self.assertContains(response, user.email)

        # test copying course permissions
        copy_course_key = SlashSeparatedCourseKey('copyu', 'copydept.mycourse', 'myrun')
        for role in [CourseInstructorRole, CourseStaffRole]:
            auth.add_users(
                self.user,
                role(copy_course_key),
                *role(self.course_key).users_with_role()
            )
        # verify access in copy course and verify that removal from source course w/ the various
        # groupnames works
        for role in [CourseInstructorRole, CourseStaffRole]:
            for user in user_by_role[role]:
                # forcefully decache the groups: premise is that any real request will not have
                # multiple objects repr the same user but this test somehow uses different instance
                # in above add_users call
                if hasattr(user, '_roles'):
                    del user._roles

                self.assertTrue(has_course_access(user, copy_course_key), "{} no copy access".format(user))
                auth.remove_users(self.user, role(self.course_key), user)
                self.assertFalse(has_course_access(user, self.course_key), "{} remove didn't work".format(user))

########NEW FILE########
__FILENAME__ = test_request_event
"""Tests for CMS's requests to logs"""
import mock

from django.test import TestCase
from django.core.urlresolvers import reverse
from contentstore.views.helpers import event as cms_user_track


class CMSLogTest(TestCase):
    """
    Tests that request to logs from CMS return 204s
    """

    def test_post_answers_to_log(self):
        """
        Checks that student answer requests submitted to cms's "/event" url
        via POST are correctly returned as 204s
        """
        requests = [
            {"event": "my_event", "event_type": "my_event_type", "page": "my_page"},
            {"event": "{'json': 'object'}", "event_type": unichr(512), "page": "my_page"}
        ]
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_SQL_TRACKING_LOGS': True}):
            for request_params in requests:
                response = self.client.post(reverse(cms_user_track), request_params)
                self.assertEqual(response.status_code, 204)

    def test_get_answers_to_log(self):
        """
        Checks that student answer requests submitted to cms's "/event" url
        via GET are correctly returned as 204s
        """
        requests = [
            {"event": "my_event", "event_type": "my_event_type", "page": "my_page"},
            {"event": "{'json': 'object'}", "event_type": unichr(512), "page": "my_page"}
        ]
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_SQL_TRACKING_LOGS': True}):
            for request_params in requests:
                response = self.client.get(reverse(cms_user_track), request_params)
                self.assertEqual(response.status_code, 204)

########NEW FILE########
__FILENAME__ = test_transcripts_utils
# -*- coding: utf-8 -*-
""" Tests for transcripts_utils. """
import unittest
from uuid import uuid4
import copy
import textwrap
from mock import patch, Mock

from pymongo import MongoClient

from django.test.utils import override_settings
from django.conf import settings
from django.utils import translation

from nose.plugins.skip import SkipTest

from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.exceptions import NotFoundError
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.video_module import transcripts_utils

from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


class TestGenerateSubs(unittest.TestCase):
    """Tests for `generate_subs` function."""
    def setUp(self):
        self.source_subs = {
            'start': [100, 200, 240, 390, 1000],
            'end': [200, 240, 380, 1000, 1500],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }

    def test_generate_subs_increase_speed(self):
        subs = transcripts_utils.generate_subs(2, 1, self.source_subs)
        self.assertDictEqual(
            subs,
            {
                'start': [200, 400, 480, 780, 2000],
                'end': [400, 480, 760, 2000, 3000],
                'text': ['subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5']
            }
        )

    def test_generate_subs_decrease_speed_1(self):
        subs = transcripts_utils.generate_subs(0.5, 1, self.source_subs)
        self.assertDictEqual(
            subs,
            {
                'start': [50, 100, 120, 195, 500],
                'end': [100, 120, 190, 500, 750],
                'text': ['subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5']
            }
        )

    def test_generate_subs_decrease_speed_2(self):
        """Test for correct devision during `generate_subs` process."""
        subs = transcripts_utils.generate_subs(1, 2, self.source_subs)
        self.assertDictEqual(
            subs,
            {
                'start': [50, 100, 120, 195, 500],
                'end': [100, 120, 190, 500, 750],
                'text': ['subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5']
            }
        )


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class TestSaveSubsToStore(ModuleStoreTestCase):
    """Tests for `save_subs_to_store` function."""

    org = 'MITx'
    number = '999'
    display_name = 'Test course'

    def clear_subs_content(self):
        """Remove, if subtitles content exists."""
        try:
            content = contentstore().find(self.content_location)
            contentstore().delete(content.get_id())
        except NotFoundError:
            pass

    def setUp(self):

        self.course = CourseFactory.create(
            org=self.org, number=self.number, display_name=self.display_name)

        self.subs = {
            'start': [100, 200, 240, 390, 1000],
            'end': [200, 240, 380, 1000, 1500],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }

        self.subs_id = str(uuid4())
        filename = 'subs_{0}.srt.sjson'.format(self.subs_id)
        self.content_location = StaticContent.compute_location(self.course.id, filename)

        # incorrect  subs
        self.unjsonable_subs = set([1])  # set can't be serialized

        self.unjsonable_subs_id = str(uuid4())
        filename_unjsonable = 'subs_{0}.srt.sjson'.format(self.unjsonable_subs_id)
        self.content_location_unjsonable = StaticContent.compute_location(self.course.id, filename_unjsonable)

        self.clear_subs_content()

    def test_save_subs_to_store(self):
        with self.assertRaises(NotFoundError):
            contentstore().find(self.content_location)

        result_location = transcripts_utils.save_subs_to_store(
            self.subs,
            self.subs_id,
            self.course)

        self.assertTrue(contentstore().find(self.content_location))
        self.assertEqual(result_location, self.content_location)

    def test_save_unjsonable_subs_to_store(self):
        """
        Assures that subs, that can't be dumped, can't be found later.
        """
        with self.assertRaises(NotFoundError):
            contentstore().find(self.content_location_unjsonable)

        with self.assertRaises(TypeError):
            transcripts_utils.save_subs_to_store(
                self.unjsonable_subs,
                self.unjsonable_subs_id,
                self.course)

        with self.assertRaises(NotFoundError):
            contentstore().find(self.content_location_unjsonable)

    def tearDown(self):
        self.clear_subs_content()
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class TestDownloadYoutubeSubs(ModuleStoreTestCase):
    """Tests for `download_youtube_subs` function."""

    org = 'MITx'
    number = '999'
    display_name = 'Test course'

    def clear_subs_content(self, youtube_subs):
        """Remove, if subtitles content exists."""
        for subs_id in youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(self.course.id, filename)
            try:
                content = contentstore().find(content_location)
                contentstore().delete(content.get_id())
            except NotFoundError:
                pass

    def setUp(self):
        self.course = CourseFactory.create(
            org=self.org, number=self.number, display_name=self.display_name)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_success_downloading_subs(self):

        response = textwrap.dedent("""<?xml version="1.0" encoding="utf-8" ?>
                <transcript>
                    <text start="0" dur="0.27"></text>
                    <text start="0.27" dur="2.45">Test text 1.</text>
                    <text start="2.72">Test text 2.</text>
                    <text start="5.43" dur="1.73">Test text 3.</text>
                </transcript>
        """)
        good_youtube_subs = {
            0.5: 'good_id_1',
            1.0: 'good_id_2',
            2.0: 'good_id_3'
        }
        self.clear_subs_content(good_youtube_subs)

        with patch('xmodule.video_module.transcripts_utils.requests.get') as mock_get:
            mock_get.return_value = Mock(status_code=200, text=response, content=response)
            # Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown
            transcripts_utils.download_youtube_subs(good_youtube_subs, self.course, settings)

        mock_get.assert_any_call('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_1'})
        mock_get.assert_any_call('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_2'})
        mock_get.assert_any_call('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_3'})

        # Check assets status after importing subtitles.
        for subs_id in good_youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(self.course.id, filename)
            self.assertTrue(contentstore().find(content_location))

        self.clear_subs_content(good_youtube_subs)

    def test_subs_for_html5_vid_with_periods(self):
        """
        This is to verify a fix whereby subtitle files uploaded against
        a HTML5 video that contains periods in the name causes
        incorrect subs name parsing
        """
        html5_ids = transcripts_utils.get_html5_ids(['foo.mp4', 'foo.1.bar.mp4', 'foo/bar/baz.1.4.mp4', 'foo'])
        self.assertEqual(4, len(html5_ids))
        self.assertEqual(html5_ids[0], 'foo')
        self.assertEqual(html5_ids[1], 'foo.1.bar')
        self.assertEqual(html5_ids[2], 'baz.1.4')
        self.assertEqual(html5_ids[3], 'foo')

    @patch('xmodule.video_module.transcripts_utils.requests.get')
    def test_fail_downloading_subs(self, mock_get):

        mock_get.return_value = Mock(status_code=404, text='Error 404')

        bad_youtube_subs = {
            0.5: 'BAD_YOUTUBE_ID1',
            1.0: 'BAD_YOUTUBE_ID2',
            2.0: 'BAD_YOUTUBE_ID3'
        }
        self.clear_subs_content(bad_youtube_subs)
        with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
            transcripts_utils.download_youtube_subs(bad_youtube_subs, self.course, settings)

        # Check assets status after importing subtitles.
        for subs_id in bad_youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(
                self.course.id, filename
            )
            with self.assertRaises(NotFoundError):
                contentstore().find(content_location)

        self.clear_subs_content(bad_youtube_subs)

    def test_success_downloading_chinese_transcripts(self):

        # Disabled 11/14/13
        # This test is flakey because it performs an HTTP request on an external service
        # Re-enable when `requests.get` is patched using `mock.patch`
        raise SkipTest

        good_youtube_subs = {
            1.0: 'j_jEn79vS3g',  # Chinese, utf-8
        }
        self.clear_subs_content(good_youtube_subs)

        # Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown
        transcripts_utils.download_youtube_subs(good_youtube_subs, self.course, settings)

        # Check assets status after importing subtitles.
        for subs_id in good_youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(
                self.course.id, filename
            )
            self.assertTrue(contentstore().find(content_location))

        self.clear_subs_content(good_youtube_subs)


class TestGenerateSubsFromSource(TestDownloadYoutubeSubs):
    """Tests for `generate_subs_from_source` function."""

    def test_success_generating_subs(self):
        youtube_subs = {
            0.5: 'JMD_ifUUfsU',
            1.0: 'hI10vDNYz4M',
            2.0: 'AKqURZnYqpk'
        }
        srt_filedata = textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """)
        self.clear_subs_content(youtube_subs)

        # Check transcripts_utils.TranscriptsGenerationException not thrown.
        # Also checks that uppercase file extensions are supported.
        transcripts_utils.generate_subs_from_source(youtube_subs, 'SRT', srt_filedata, self.course)

        # Check assets status after importing subtitles.
        for subs_id in youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(
                self.course.id, filename
            )
            self.assertTrue(contentstore().find(content_location))

        self.clear_subs_content(youtube_subs)

    def test_fail_bad_subs_type(self):
        youtube_subs = {
            0.5: 'JMD_ifUUfsU',
            1.0: 'hI10vDNYz4M',
            2.0: 'AKqURZnYqpk'
        }

        srt_filedata = textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """)

        with self.assertRaises(transcripts_utils.TranscriptsGenerationException) as cm:
            transcripts_utils.generate_subs_from_source(youtube_subs, 'BAD_FORMAT', srt_filedata, self.course)
        exception_message = cm.exception.message
        self.assertEqual(exception_message, "We support only SubRip (*.srt) transcripts format.")

    def test_fail_bad_subs_filedata(self):
        youtube_subs = {
            0.5: 'JMD_ifUUfsU',
            1.0: 'hI10vDNYz4M',
            2.0: 'AKqURZnYqpk'
        }

        srt_filedata = """BAD_DATA"""

        with self.assertRaises(transcripts_utils.TranscriptsGenerationException) as cm:
            transcripts_utils.generate_subs_from_source(youtube_subs, 'srt', srt_filedata, self.course)
        exception_message = cm.exception.message
        self.assertEqual(exception_message, "Something wrong with SubRip transcripts file during parsing.")


class TestGenerateSrtFromSjson(TestDownloadYoutubeSubs):
    """Tests for `generate_srt_from_sjson` function."""

    def test_success_generating_subs(self):
        sjson_subs = {
            'start': [100, 200, 240, 390, 54000],
            'end': [200, 240, 380, 1000, 78400],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 1)
        self.assertTrue(srt_subs)
        expected_subs = [
            '00:00:00,100 --> 00:00:00,200\nsubs #1',
            '00:00:00,200 --> 00:00:00,240\nsubs #2',
            '00:00:00,240 --> 00:00:00,380\nsubs #3',
            '00:00:00,390 --> 00:00:01,000\nsubs #4',
            '00:00:54,000 --> 00:01:18,400\nsubs #5',
        ]

        for sub in expected_subs:
            self.assertIn(sub, srt_subs)

    def test_success_generating_subs_speed_up(self):
        sjson_subs = {
            'start': [100, 200, 240, 390, 54000],
            'end': [200, 240, 380, 1000, 78400],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 0.5)
        self.assertTrue(srt_subs)
        expected_subs = [
            '00:00:00,050 --> 00:00:00,100\nsubs #1',
            '00:00:00,100 --> 00:00:00,120\nsubs #2',
            '00:00:00,120 --> 00:00:00,190\nsubs #3',
            '00:00:00,195 --> 00:00:00,500\nsubs #4',
            '00:00:27,000 --> 00:00:39,200\nsubs #5',
        ]
        for sub in expected_subs:
            self.assertIn(sub, srt_subs)

    def test_success_generating_subs_speed_down(self):
        sjson_subs = {
            'start': [100, 200, 240, 390, 54000],
            'end': [200, 240, 380, 1000, 78400],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 2)
        self.assertTrue(srt_subs)

        expected_subs = [
            '00:00:00,200 --> 00:00:00,400\nsubs #1',
            '00:00:00,400 --> 00:00:00,480\nsubs #2',
            '00:00:00,480 --> 00:00:00,760\nsubs #3',
            '00:00:00,780 --> 00:00:02,000\nsubs #4',
            '00:01:48,000 --> 00:02:36,800\nsubs #5',
        ]
        for sub in expected_subs:
            self.assertIn(sub, srt_subs)

    def test_fail_generating_subs(self):
        sjson_subs = {
            'start': [100, 200],
            'end': [100],
            'text': [
                'subs #1',
                'subs #2'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 1)
        self.assertFalse(srt_subs)


class TestYoutubeTranscripts(unittest.TestCase):
    """
    Tests for checking right datastructure returning when using youtube api.
    """
    @patch('xmodule.video_module.transcripts_utils.requests.get')
    def test_youtube_bad_status_code(self, mock_get):
        mock_get.return_value = Mock(status_code=404, text='test')
        youtube_id = 'bad_youtube_id'
        with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
            transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation)

    @patch('xmodule.video_module.transcripts_utils.requests.get')
    def test_youtube_empty_text(self, mock_get):
        mock_get.return_value = Mock(status_code=200, text='')
        youtube_id = 'bad_youtube_id'
        with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
            transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation)

    def test_youtube_good_result(self):
        response = textwrap.dedent("""<?xml version="1.0" encoding="utf-8" ?>
                <transcript>
                    <text start="0" dur="0.27"></text>
                    <text start="0.27" dur="2.45">Test text 1.</text>
                    <text start="2.72">Test text 2.</text>
                    <text start="5.43" dur="1.73">Test text 3.</text>
                </transcript>
        """)
        expected_transcripts = {
            'start': [270, 2720, 5430],
            'end': [2720, 2720, 7160],
            'text': ['Test text 1.', 'Test text 2.', 'Test text 3.']
        }
        youtube_id = 'good_youtube_id'
        with patch('xmodule.video_module.transcripts_utils.requests.get') as mock_get:
            mock_get.return_value = Mock(status_code=200, text=response, content=response)
            transcripts = transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation)
        self.assertEqual(transcripts, expected_transcripts)
        mock_get.assert_called_with('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_youtube_id'})

class TestTranscript(unittest.TestCase):
    """
    Tests for Transcript class e.g. different transcript conversions.
    """
    def setUp(self):

        self.srt_transcript = textwrap.dedent("""\
            0
            00:00:10,500 --> 00:00:13,000
            Elephant&#39;s Dream

            1
            00:00:15,000 --> 00:00:18,000
            At the left we can see...

        """)


        self.sjson_transcript = textwrap.dedent("""\
            {
                "start": [
                    10500,
                    15000
                ],
                "end": [
                    13000,
                    18000
                ],
                "text": [
                    "Elephant&#39;s Dream",
                    "At the left we can see..."
                ]
            }
        """)

        self.txt_transcript = u"Elephant's Dream\nAt the left we can see..."

    def test_convert_srt_to_txt(self):
        expected = self.txt_transcript
        actual = transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'txt')
        self.assertEqual(actual, expected)

    def test_convert_srt_to_srt(self):
        expected = self.srt_transcript
        actual = transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'srt')
        self.assertEqual(actual, expected)

    def test_convert_sjson_to_txt(self):
        expected = self.txt_transcript
        actual = transcripts_utils.Transcript.convert(self.sjson_transcript, 'sjson', 'txt')
        self.assertEqual(actual, expected)

    def test_convert_sjson_to_srt(self):
        expected = self.srt_transcript
        actual = transcripts_utils.Transcript.convert(self.sjson_transcript, 'sjson', 'srt')
        self.assertEqual(actual, expected)

    def test_convert_srt_to_sjson(self):
        with self.assertRaises(NotImplementedError):
            transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'sjson')


class TestSubsFilename(unittest.TestCase):
    """
    Tests for subs_filename funtion.
    """

    def test_unicode(self):
        name = transcripts_utils.subs_filename(u"˙∆©ƒƒƒ")
        self.assertEqual(name, u'subs_˙∆©ƒƒƒ.srt.sjson')
        name = transcripts_utils.subs_filename(u"˙∆©ƒƒƒ", 'uk')
        self.assertEqual(name, u'uk_subs_˙∆©ƒƒƒ.srt.sjson')

########NEW FILE########
__FILENAME__ = test_users_default_role
"""
Unit tests for checking default forum role "Student" of a user when he creates a course or
after deleting it creates same course again
"""
from contentstore.tests.utils import AjaxEnabledTestClient
from contentstore.utils import delete_course_and_groups, reverse_url
from courseware.tests.factories import UserFactory
from xmodule.modulestore import Location
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.locations import SlashSeparatedCourseKey

from student.models import CourseEnrollment


class TestUsersDefaultRole(ModuleStoreTestCase):
    """
    Unit tests for checking enrollment and default forum role "Student" of a logged in user
    """
    def setUp(self):
        """
        Add a user and a course
        """
        super(TestUsersDefaultRole, self).setUp()
        # create and log in a staff user.
        self.user = UserFactory(is_staff=True)  # pylint: disable=no-member
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.user.username, password='test')

        # create a course via the view handler to create course
        self.course_key = SlashSeparatedCourseKey('Org_1', 'Course_1', 'Run_1')
        self._create_course_with_given_location(self.course_key)

    def _create_course_with_given_location(self, course_key):
        """
        Create course at provided location
        """
        resp = self.client.ajax_post(
            reverse_url('course_handler'),
            {
                'org': course_key.org,
                'number': course_key.course,
                'display_name': 'test course',
                'run': course_key.run,
            }
        )
        return resp

    def tearDown(self):
        """
        Reverse the setup
        """
        self.client.logout()
        super(TestUsersDefaultRole, self).tearDown()

    def test_user_forum_default_role_on_course_deletion(self):
        """
        Test that a user enrolls and gets "Student" forum role for that course which he creates and remains
        enrolled even the course is deleted and keeps its "Student" forum role for that course
        """
        # check that user has enrollment for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        # check that user has his default "Student" forum role for this course
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

        delete_course_and_groups(self.course_key, commit=True)

        # check that user's enrollment for this course is not deleted
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        # check that user has forum role for this course even after deleting it
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

    def test_user_role_on_course_recreate(self):
        """
        Test that creating same course again after deleting it gives user his default
        forum role "Student" for that course
        """
        # check that user has enrollment and his default "Student" forum role for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

        # delete this course and recreate this course with same user
        delete_course_and_groups(self.course_key, commit=True)
        resp = self._create_course_with_given_location(self.course_key)
        self.assertEqual(resp.status_code, 200)

        # check that user has his enrollment for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        # check that user has his default "Student" forum role for this course
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

    def test_user_role_on_course_recreate_with_change_name_case(self):
        """
        Test that creating same course again with different name case after deleting it gives user
        his default forum role "Student" for that course
        """
        # check that user has enrollment and his default "Student" forum role for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))
        # delete this course and recreate this course with same user
        delete_course_and_groups(self.course_key, commit=True)

        # now create same course with different name case ('uppercase')
        new_course_key = self.course_key.replace(course=self.course_key.course.upper())
        resp = self._create_course_with_given_location(new_course_key)
        self.assertEqual(resp.status_code, 200)

        # check that user has his default "Student" forum role again for this course (with changed name case)
        self.assertTrue(
            self.user.roles.filter(name="Student", course_id=new_course_key)  # pylint: disable=no-member
        )

        # Disabled due to case-sensitive test db (sqlite3)
        # # check that there user has only one "Student" forum role (with new updated course_id)
        # self.assertEqual(self.user.roles.filter(name='Student').count(), 1)  # pylint: disable=no-member
        # self.assertEqual(self.user.roles.filter(name='Student')[0].course_id, new_course_location.course_key)

########NEW FILE########
__FILENAME__ = test_utils
""" Tests for utils. """
import collections
import copy
import mock

from django.test import TestCase
from django.test.utils import override_settings

from contentstore import utils
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class LMSLinksTestCase(TestCase):
    """ Tests for LMS links. """
    def about_page_test(self):
        """ Get URL for about page, no marketing site """
        # default for ENABLE_MKTG_SITE is False.
        self.assertEquals(self.get_about_page_link(), "//localhost:8000/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'dummy-root'})
    def about_page_marketing_site_test(self):
        """ Get URL for about page, marketing root present. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//dummy-root/courses/mitX/101/test/about")
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': False}):
            self.assertEquals(self.get_about_page_link(), "//localhost:8000/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'http://www.dummy'})
    def about_page_marketing_site_remove_http_test(self):
        """ Get URL for about page, marketing root present, remove http://. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//www.dummy/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'https://www.dummy'})
    def about_page_marketing_site_remove_https_test(self):
        """ Get URL for about page, marketing root present, remove https://. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//www.dummy/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'www.dummyhttps://x'})
    def about_page_marketing_site_https__edge_test(self):
        """ Get URL for about page, only remove https:// at the beginning of the string. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//www.dummyhttps://x/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={})
    def about_page_marketing_urls_not_set_test(self):
        """ Error case. ENABLE_MKTG_SITE is True, but there is either no MKTG_URLS, or no MKTG_URLS Root property. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), None)

    @override_settings(LMS_BASE=None)
    def about_page_no_lms_base_test(self):
        """ No LMS_BASE, nor is ENABLE_MKTG_SITE True """
        self.assertEquals(self.get_about_page_link(), None)

    def get_about_page_link(self):
        """ create mock course and return the about page link """
        course_key = SlashSeparatedCourseKey('mitX', '101', 'test')
        return utils.get_lms_link_for_about_page(course_key)

    def lms_link_test(self):
        """ Tests get_lms_link_for_item. """
        course_key = SlashSeparatedCourseKey('mitX', '101', 'test')
        location = course_key.make_usage_key('vertical', 'contacting_us')
        link = utils.get_lms_link_for_item(location, False)
        self.assertEquals(link, "//localhost:8000/courses/mitX/101/test/jump_to/i4x://mitX/101/vertical/contacting_us")

        # test preview
        link = utils.get_lms_link_for_item(location, True)
        self.assertEquals(
            link,
            "//preview/courses/mitX/101/test/jump_to/i4x://mitX/101/vertical/contacting_us"
        )

        # now test with the course' location
        location = course_key.make_usage_key('course', 'test')
        link = utils.get_lms_link_for_item(location)
        self.assertEquals(link, "//localhost:8000/courses/mitX/101/test/jump_to/i4x://mitX/101/course/test")

class ExtraPanelTabTestCase(TestCase):
    """ Tests adding and removing extra course tabs. """

    def get_tab_type_dicts(self, tab_types):
        """ Returns an array of tab dictionaries. """
        if tab_types:
            return [{'tab_type': tab_type} for tab_type in tab_types.split(',')]
        else:
            return []

    def get_course_with_tabs(self, tabs=None):
        """ Returns a mock course object with a tabs attribute. """
        if tabs is None:
            tabs = []
        course = collections.namedtuple('MockCourse', ['tabs'])
        if isinstance(tabs, basestring):
            course.tabs = self.get_tab_type_dicts(tabs)
        else:
            course.tabs = tabs
        return course

    def test_add_extra_panel_tab(self):
        """ Tests if a tab can be added to a course tab list. """
        for tab_type in utils.EXTRA_TAB_PANELS.keys():
            tab = utils.EXTRA_TAB_PANELS.get(tab_type)

            # test adding with changed = True
            for tab_setup in ['', 'x', 'x,y,z']:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = copy.copy(course.tabs)
                expected_tabs.append(tab)
                changed, actual_tabs = utils.add_extra_panel_tab(tab_type, course)
                self.assertTrue(changed)
                self.assertEqual(actual_tabs, expected_tabs)

            # test adding with changed = False
            tab_test_setup = [
                [tab],
                [tab, self.get_tab_type_dicts('x,y,z')],
                [self.get_tab_type_dicts('x,y'), tab, self.get_tab_type_dicts('z')],
                [self.get_tab_type_dicts('x,y,z'), tab]]

            for tab_setup in tab_test_setup:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = copy.copy(course.tabs)
                changed, actual_tabs = utils.add_extra_panel_tab(tab_type, course)
                self.assertFalse(changed)
                self.assertEqual(actual_tabs, expected_tabs)

    def test_remove_extra_panel_tab(self):
        """ Tests if a tab can be removed from a course tab list. """
        for tab_type in utils.EXTRA_TAB_PANELS.keys():
            tab = utils.EXTRA_TAB_PANELS.get(tab_type)

            # test removing with changed = True
            tab_test_setup = [
                [tab],
                [tab, self.get_tab_type_dicts('x,y,z')],
                [self.get_tab_type_dicts('x,y'), tab, self.get_tab_type_dicts('z')],
                [self.get_tab_type_dicts('x,y,z'), tab]]

            for tab_setup in tab_test_setup:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = [t for t in course.tabs if t != utils.EXTRA_TAB_PANELS.get(tab_type)]
                changed, actual_tabs = utils.remove_extra_panel_tab(tab_type, course)
                self.assertTrue(changed)
                self.assertEqual(actual_tabs, expected_tabs)

            # test removing with changed = False
            for tab_setup in ['', 'x', 'x,y,z']:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = copy.copy(course.tabs)
                changed, actual_tabs = utils.remove_extra_panel_tab(tab_type, course)
                self.assertFalse(changed)
                self.assertEqual(actual_tabs, expected_tabs)


class CourseImageTestCase(TestCase):
    """Tests for course image URLs."""

    def test_get_image_url(self):
        """Test image URL formatting."""
        course = CourseFactory.create(org='edX', course='999')
        url = utils.course_image_url(course)
        self.assertEquals(url, '/c4x/edX/999/asset/{0}'.format(course.course_image))

    def test_non_ascii_image_name(self):
        # Verify that non-ascii image names are cleaned
        course = CourseFactory.create(course_image=u'before_\N{SNOWMAN}_after.jpg')
        self.assertEquals(
            utils.course_image_url(course),
            '/c4x/{org}/{course}/asset/before___after.jpg'.format(org=course.location.org, course=course.location.course)
        )

    def test_spaces_in_image_name(self):
        # Verify that image names with spaces in them are cleaned
        course = CourseFactory.create(course_image=u'before after.jpg')
        self.assertEquals(
            utils.course_image_url(course),
            '/c4x/{org}/{course}/asset/before_after.jpg'.format(
                org=course.location.org,
                course=course.location.course
            )
        )

########NEW FILE########
__FILENAME__ = utils
'''
Utilities for contentstore tests
'''

import json

from django.contrib.auth.models import User
from django.test.client import Client
from django.test.utils import override_settings

from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.utils import get_modulestore
from student.models import Registration


def parse_json(response):
    """Parse response, which is assumed to be json"""
    return json.loads(response.content)


def user(email):
    """look up a user by email"""
    return User.objects.get(email=email)


def registration(email):
    """look up registration object by email"""
    return Registration.objects.get(user__email=email)


class AjaxEnabledTestClient(Client):
    """
    Convenience class to make testing easier.
    """
    def ajax_post(self, path, data=None, content_type="application/json", **kwargs):
        """
        Convenience method for client post which serializes the data into json and sets the accept type
        to json
        """
        if not isinstance(data, basestring):
            data = json.dumps(data or {})
        kwargs.setdefault("HTTP_X_REQUESTED_WITH", "XMLHttpRequest")
        kwargs.setdefault("HTTP_ACCEPT", "application/json")
        return self.post(path=path, data=data, content_type=content_type, **kwargs)

    def get_html(self, path, data=None, follow=False, **extra):
        """
        Convenience method for client.get which sets the accept type to html
        """
        return self.get(path, data or {}, follow, HTTP_ACCEPT="text/html", **extra)

    def get_json(self, path, data=None, follow=False, **extra):
        """
        Convenience method for client.get which sets the accept type to json
        """
        return self.get(path, data or {}, follow, HTTP_ACCEPT="application/json", **extra)



@override_settings(MODULESTORE=TEST_MODULESTORE)
class CourseTestCase(ModuleStoreTestCase):
    def setUp(self):
        """
        These tests need a user in the DB so that the django Test Client
        can log them in.
        They inherit from the ModuleStoreTestCase class so that the mongodb collection
        will be cleared out before each test case execution and deleted
        afterwards.
        """
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the user so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

        self.course = CourseFactory.create(
            org='MITx',
            number='999',
            display_name='Robot Super Course',
        )
        self.store = get_modulestore(self.course.location)

    def create_non_staff_authed_user_client(self, authenticate=True):
        """
        Create a non-staff user, log them in (if authenticate=True), and return the client, user to use for testing.
        """
        uname = 'teststudent'
        password = 'foo'
        nonstaff = User.objects.create_user(uname, 'test+student@edx.org', password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        nonstaff.is_active = True
        nonstaff.is_staff = False
        nonstaff.save()

        client = Client()
        if authenticate:
            client.login(username=uname, password=password)
        return client, nonstaff

    def populate_course(self):
        """
        Add 2 chapters, 4 sections, 8 verticals, 16 problems to self.course (branching 2)
        """
        def descend(parent, stack):
            xblock_type = stack.pop(0)
            for _ in range(2):
                child = ItemFactory.create(category=xblock_type, parent_location=parent.location)
                if stack:
                    descend(child, stack)

        descend(self.course, ['chapter', 'sequential', 'vertical', 'problem'])

    def reload_course(self):
        """
        Reloads the course object from the database
        """
        self.course = self.store.get_course(self.course.id)

    def save_course(self):
        """
        Updates the course object in the database
        """
        self.course.save()
        self.store.update_item(self.course, self.user.id)

########NEW FILE########
__FILENAME__ = utils
# pylint: disable=E1103, E1101

import copy
import logging
import re

from django.conf import settings
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse

from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore.store_utilities import delete_course
from student.roles import CourseInstructorRole, CourseStaffRole


log = logging.getLogger(__name__)

# In order to instantiate an open ended tab automatically, need to have this data
OPEN_ENDED_PANEL = {"name": _("Open Ended Panel"), "type": "open_ended"}
NOTES_PANEL = {"name": _("My Notes"), "type": "notes"}
EXTRA_TAB_PANELS = dict([(p['type'], p) for p in [OPEN_ENDED_PANEL, NOTES_PANEL]])


def delete_course_and_groups(course_id, commit=False):
    """
    This deletes the courseware associated with a course_id as well as cleaning update_item
    the various user table stuff (groups, permissions, etc.)
    """
    module_store = modulestore('direct')
    content_store = contentstore()

    module_store.ignore_write_events_on_courses.add(course_id)

    if delete_course(module_store, content_store, course_id, commit):

        print 'removing User permissions from course....'
        # in the django layer, we need to remove all the user permissions groups associated with this course
        if commit:
            try:
                staff_role = CourseStaffRole(course_id)
                staff_role.remove_users(*staff_role.users_with_role())
                instructor_role = CourseInstructorRole(course_id)
                instructor_role.remove_users(*instructor_role.users_with_role())
            except Exception as err:
                log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))


def get_modulestore(category_or_location):
    """
    This function no longer does anything more than just calling `modulestore()`. It used
    to select 'direct' v 'draft' based on the category.
    """
    return modulestore()


def get_lms_link_for_item(location, preview=False):
    """
    Returns an LMS link to the course with a jump_to to the provided location.

    :param location: the location to jump to
    :param preview: True if the preview version of LMS should be returned. Default value is false.
    """
    assert(isinstance(location, Location))

    if settings.LMS_BASE is None:
        return None

    if preview:
        lms_base = settings.FEATURES.get('PREVIEW_LMS_BASE')
    else:
        lms_base = settings.LMS_BASE

    return u"//{lms_base}/courses/{course_id}/jump_to/{location}".format(
        lms_base=lms_base,
        course_id=location.course_key.to_deprecated_string(),
        location=location.to_deprecated_string(),
    )


def get_lms_link_for_about_page(course_id):
    """
    Returns the url to the course about page from the location tuple.
    """

    assert(isinstance(course_id, SlashSeparatedCourseKey))

    if settings.FEATURES.get('ENABLE_MKTG_SITE', False):
        if not hasattr(settings, 'MKTG_URLS'):
            log.exception("ENABLE_MKTG_SITE is True, but MKTG_URLS is not defined.")
            return None

        marketing_urls = settings.MKTG_URLS

        # Root will be "https://www.edx.org". The complete URL will still not be exactly correct,
        # but redirects exist from www.edx.org to get to the Drupal course about page URL.
        about_base = marketing_urls.get('ROOT', None)

        if about_base is None:
            log.exception('There is no ROOT defined in MKTG_URLS')
            return None

        # Strip off https:// (or http://) to be consistent with the formatting of LMS_BASE.
        about_base = re.sub(r"^https?://", "", about_base)

    elif settings.LMS_BASE is not None:
        about_base = settings.LMS_BASE
    else:
        return None

    return u"//{about_base_url}/courses/{course_id}/about".format(
        about_base_url=about_base,
        course_id=course_id.to_deprecated_string()
    )


def course_image_url(course):
    """Returns the image url for the course."""
    loc = StaticContent.compute_location(course.location.course_key, course.course_image)
    path = loc.to_deprecated_string()
    return path


class PublishState(object):
    """
    The publish state for a given xblock-- either 'draft', 'private', or 'public'.

    Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
    """
    draft = 'draft'
    private = 'private'
    public = 'public'


def compute_publish_state(xblock):
    """
    Returns whether this xblock is 'draft', 'public', or 'private'.

    'draft' content is in the process of being edited, but still has a previous
        version visible in the LMS
    'public' content is locked and visible in the LMS
    'private' content is editable and not visible in the LMS
    """

    if getattr(xblock, 'is_draft', False):
        try:
            modulestore('direct').get_item(xblock.location)
            return PublishState.draft
        except ItemNotFoundError:
            return PublishState.private
    else:
        return PublishState.public


def add_extra_panel_tab(tab_type, course):
    """
    Used to add the panel tab to a course if it does not exist.
    @param tab_type: A string representing the tab type.
    @param course: A course object from the modulestore.
    @return: Boolean indicating whether or not a tab was added and a list of tabs for the course.
    """
    # Copy course tabs
    course_tabs = copy.copy(course.tabs)
    changed = False
    # Check to see if open ended panel is defined in the course

    tab_panel = EXTRA_TAB_PANELS.get(tab_type)
    if tab_panel not in course_tabs:
        # Add panel to the tabs if it is not defined
        course_tabs.append(tab_panel)
        changed = True
    return changed, course_tabs


def remove_extra_panel_tab(tab_type, course):
    """
    Used to remove the panel tab from a course if it exists.
    @param tab_type: A string representing the tab type.
    @param course: A course object from the modulestore.
    @return: Boolean indicating whether or not a tab was added and a list of tabs for the course.
    """
    # Copy course tabs
    course_tabs = copy.copy(course.tabs)
    changed = False
    # Check to see if open ended panel is defined in the course

    tab_panel = EXTRA_TAB_PANELS.get(tab_type)
    if tab_panel in course_tabs:
        # Add panel to the tabs if it is not defined
        course_tabs = [ct for ct in course_tabs if ct != tab_panel]
        changed = True
    return changed, course_tabs


def reverse_url(handler_name, key_name=None, key_value=None, kwargs=None):
    """
    Creates the URL for the given handler.
    The optional key_name and key_value are passed in as kwargs to the handler.
    """
    kwargs_for_reverse = {key_name: unicode(key_value)} if key_name else None
    if kwargs:
        kwargs_for_reverse.update(kwargs)
    return reverse('contentstore.views.' + handler_name, kwargs=kwargs_for_reverse)


def reverse_course_url(handler_name, course_key, kwargs=None):
    """
    Creates the URL for handlers that use course_keys as URL parameters.
    """
    return reverse_url(handler_name, 'course_key_string', course_key, kwargs)


def reverse_usage_url(handler_name, usage_key, kwargs=None):
    """
    Creates the URL for handlers that use usage_keys as URL parameters.
    """
    return reverse_url(handler_name, 'usage_key_string', usage_key, kwargs)

########NEW FILE########
__FILENAME__ = access
from student.roles import CourseStaffRole, GlobalStaff, CourseInstructorRole
from student import auth


def has_course_access(user, course_key, role=CourseStaffRole):
    """
    Return True if user allowed to access this course_id
    Note that the CMS permissions model is with respect to courses
    There is a super-admin permissions if user.is_staff is set
    Also, since we're unifying the user database between LMS and CAS,
    I'm presuming that the course instructor (formally known as admin)
    will not be in both INSTRUCTOR and STAFF groups, so we have to cascade our
    queries here as INSTRUCTOR has all the rights that STAFF do
    """
    if GlobalStaff().has_user(user):
        return True
    return auth.has_access(user, role(course_key))


def get_user_role(user, course_id):
    """
    What type of access: staff or instructor does this user have in Studio?

    No code should use this for access control, only to quickly serialize the type of access
    where this code knows that Instructor trumps Staff and assumes the user has one or the other.

    This will not return student role because its purpose for using in Studio.

    :param course_id: the course_id of the course we're interested in
    """
    # afaik, this is only used in lti
    if auth.has_access(user, CourseInstructorRole(course_id)):
        return 'instructor'
    else:
        return 'staff'

########NEW FILE########
__FILENAME__ = assets
import logging
from functools import partial
import math
import json

from django.http import HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django_future.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_POST
from django.conf import settings

from edxmako.shortcuts import render_to_response
from cache_toolbox.core import del_cached_content

from contentstore.utils import reverse_course_url
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from django.core.exceptions import PermissionDenied
from xmodule.modulestore.keys import CourseKey, AssetKey

from util.date_utils import get_default_time_display
from util.json_request import JsonResponse
from django.http import HttpResponseNotFound
from django.utils.translation import ugettext as _
from pymongo import ASCENDING, DESCENDING
from .access import has_course_access
from xmodule.modulestore.exceptions import ItemNotFoundError

__all__ = ['assets_handler']


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
def assets_handler(request, course_key_string=None, asset_key_string=None):
    """
    The restful handler for assets.
    It allows retrieval of all the assets (as an HTML page), as well as uploading new assets,
    deleting assets, and changing the "locked" state of an asset.

    GET
        html: return an html page which will show all course assets. Note that only the asset container
            is returned and that the actual assets are filled in with a client-side request.
        json: returns a page of assets. The following parameters are supported:
            page: the desired page of results (defaults to 0)
            page_size: the number of items per page (defaults to 50)
            sort: the asset field to sort by (defaults to "date_added")
            direction: the sort direction (defaults to "descending")
    POST
        json: create (or update?) an asset. The only updating that can be done is changing the lock state.
    PUT
        json: update the locked state of an asset
    DELETE
        json: delete an asset
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    response_format = request.REQUEST.get('format', 'html')
    if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            return _assets_json(request, course_key)
        else:
            asset_key = AssetKey.from_string(asset_key_string) if asset_key_string else None
            return _update_asset(request, course_key, asset_key)
    elif request.method == 'GET':  # assume html
        return _asset_index(request, course_key)
    else:
        return HttpResponseNotFound()


def _asset_index(request, course_key):
    """
    Display an editable asset library.

    Supports start (0-based index into the list of assets) and max query parameters.
    """
    course_module = modulestore().get_course(course_key)

    return render_to_response('asset_index.html', {
        'context_course': course_module,
        'asset_callback_url': reverse_course_url('assets_handler', course_key)
    })


def _assets_json(request, course_key):
    """
    Display an editable asset library.

    Supports start (0-based index into the list of assets) and max query parameters.
    """
    requested_page = int(request.REQUEST.get('page', 0))
    requested_page_size = int(request.REQUEST.get('page_size', 50))
    requested_sort = request.REQUEST.get('sort', 'date_added')
    sort_direction = DESCENDING
    if request.REQUEST.get('direction', '').lower() == 'asc':
        sort_direction = ASCENDING

    # Convert the field name to the Mongo name
    if requested_sort == 'date_added':
        requested_sort = 'uploadDate'
    elif requested_sort == 'display_name':
        requested_sort = 'displayname'
    sort = [(requested_sort, sort_direction)]

    current_page = max(requested_page, 0)
    start = current_page * requested_page_size
    assets, total_count = _get_assets_for_page(request, course_key, current_page, requested_page_size, sort)
    end = start + len(assets)

    # If the query is beyond the final page, then re-query the final page so that at least one asset is returned
    if requested_page > 0 and start >= total_count:
        current_page = int(math.floor((total_count - 1) / requested_page_size))
        start = current_page * requested_page_size
        assets, total_count = _get_assets_for_page(request, course_key, current_page, requested_page_size, sort)
        end = start + len(assets)

    asset_json = []
    for asset in assets:
        asset_id = asset['_id']
        asset_location = StaticContent.compute_location(course_key, asset_id['name'])
        # note, due to the schema change we may not have a 'thumbnail_location' in the result set
        thumbnail_location = asset.get('thumbnail_location', None)
        if thumbnail_location:
            thumbnail_location = course_key.make_asset_key('thumbnail', thumbnail_location[4])

        asset_locked = asset.get('locked', False)
        asset_json.append(_get_asset_json(asset['displayname'], asset['uploadDate'], asset_location, thumbnail_location, asset_locked))

    return JsonResponse({
        'start': start,
        'end': end,
        'page': current_page,
        'pageSize': requested_page_size,
        'totalCount': total_count,
        'assets': asset_json,
        'sort': requested_sort,
    })


def _get_assets_for_page(request, course_key, current_page, page_size, sort):
    """
    Returns the list of assets for the specified page and page size.
    """
    start = current_page * page_size

    return contentstore().get_all_content_for_course(
        course_key, start=start, maxresults=page_size, sort=sort
    )


@require_POST
@ensure_csrf_cookie
@login_required
def _upload_asset(request, course_key):
    '''
    This method allows for POST uploading of files into the course asset
    library, which will be supported by GridFS in MongoDB.
    '''
    # Does the course actually exist?!? Get anything from it to prove its
    # existence
    try:
        modulestore().get_course(course_key)
    except ItemNotFoundError:
        # no return it as a Bad Request response
        logging.error("Could not find course: %s", course_key)
        return HttpResponseBadRequest()

    # compute a 'filename' which is similar to the location formatting, we're
    # using the 'filename' nomenclature since we're using a FileSystem paradigm
    # here. We're just imposing the Location string formatting expectations to
    # keep things a bit more consistent
    upload_file = request.FILES['file']
    filename = upload_file.name
    mime_type = upload_file.content_type

    content_loc = StaticContent.compute_location(course_key, filename)

    chunked = upload_file.multiple_chunks()
    sc_partial = partial(StaticContent, content_loc, filename, mime_type)
    if chunked:
        content = sc_partial(upload_file.chunks())
        tempfile_path = upload_file.temporary_file_path()
    else:
        content = sc_partial(upload_file.read())
        tempfile_path = None

    # first let's see if a thumbnail can be created
    (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
            content,
            tempfile_path=tempfile_path
    )

    # delete cached thumbnail even if one couldn't be created this time (else
    # the old thumbnail will continue to show)
    del_cached_content(thumbnail_location)
    # now store thumbnail location only if we could create it
    if thumbnail_content is not None:
        content.thumbnail_location = thumbnail_location

    # then commit the content
    contentstore().save(content)
    del_cached_content(content.location)

    # readback the saved content - we need the database timestamp
    readback = contentstore().find(content.location)

    locked = getattr(content, 'locked', False)
    response_payload = {
        'asset': _get_asset_json(content.name, readback.last_modified_at, content.location, content.thumbnail_location, locked),
        'msg': _('Upload completed')
    }

    return JsonResponse(response_payload)


@require_http_methods(("DELETE", "POST", "PUT"))
@login_required
@ensure_csrf_cookie
def _update_asset(request, course_key, asset_key):
    """
    restful CRUD operations for a course asset.
    Currently only DELETE, POST, and PUT methods are implemented.

    asset_path_encoding: the odd /c4x/org/course/category/name repr of the asset (used by Backbone as the id)
    """
    if request.method == 'DELETE':
        # Make sure the item to delete actually exists.
        try:
            content = contentstore().find(asset_key)
        except NotFoundError:
            return JsonResponse(status=404)

        # ok, save the content into the trashcan
        contentstore('trashcan').save(content)

        # see if there is a thumbnail as well, if so move that as well
        if content.thumbnail_location is not None:
            # We are ignoring the value of the thumbnail_location-- we only care whether
            # or not a thumbnail has been stored, and we can now easily create the correct path.
            thumbnail_location = course_key.make_asset_key('thumbnail', asset_key.name)
            try:
                thumbnail_content = contentstore().find(thumbnail_location)
                contentstore('trashcan').save(thumbnail_content)
                # hard delete thumbnail from origin
                contentstore().delete(thumbnail_content.get_id())
                # remove from any caching
                del_cached_content(thumbnail_location)
            except:
                logging.warning('Could not delete thumbnail: %s', thumbnail_location)

        # delete the original
        contentstore().delete(content.get_id())
        # remove from cache
        del_cached_content(content.location)
        return JsonResponse()

    elif request.method in ('PUT', 'POST'):
        if 'file' in request.FILES:
            return _upload_asset(request, course_key)
        else:
            # Update existing asset
            try:
                modified_asset = json.loads(request.body)
            except ValueError:
                return HttpResponseBadRequest()
            contentstore().set_attr(asset_key, 'locked', modified_asset['locked'])
            # Delete the asset from the cache so we check the lock status the next time it is requested.
            del_cached_content(asset_key)
            return JsonResponse(modified_asset, status=201)


def _get_asset_json(display_name, date, location, thumbnail_location, locked):
    """
    Helper method for formatting the asset information to send to client.
    """
    asset_url = location.to_deprecated_string()
    external_url = settings.LMS_BASE + asset_url
    return {
        'display_name': display_name,
        'date_added': get_default_time_display(date),
        'url': asset_url,
        'external_url': external_url,
        'portable_url': StaticContent.get_static_path_from_location(location),
        'thumbnail': thumbnail_location.to_deprecated_string() if thumbnail_location is not None else None,
        'locked': locked,
        # Needed for Backbone delete/update.
        'id': unicode(location)
    }

########NEW FILE########
__FILENAME__ = checklist
import json
import copy

from util.json_request import JsonResponse
from django.http import HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django_future.csrf import ensure_csrf_cookie
from edxmako.shortcuts import render_to_response
from django.http import HttpResponseNotFound
from django.core.exceptions import PermissionDenied
from xmodule.modulestore.keys import CourseKey
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url

from .access import has_course_access
from xmodule.course_module import CourseDescriptor

from django.utils.translation import ugettext

__all__ = ['checklists_handler']


# pylint: disable=unused-argument
@require_http_methods(("GET", "POST", "PUT"))
@login_required
@ensure_csrf_cookie
def checklists_handler(request, course_key_string, checklist_index=None):
    """
    The restful handler for checklists.

    GET
        html: return html page for all checklists
        json: return json representing all checklists. checklist_index is not supported for GET at this time.
    POST or PUT
        json: updates the checked state for items within a particular checklist. checklist_index is required.
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)

    json_request = 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json')
    if request.method == 'GET':
        # If course was created before checklists were introduced, copy them over
        # from the template.
        if not course_module.checklists:
            course_module.checklists = CourseDescriptor.checklists.default
            get_modulestore(course_module.location).update_item(course_module, request.user.id)

        expanded_checklists = expand_all_action_urls(course_module)
        if json_request:
            return JsonResponse(expanded_checklists)
        else:
            handler_url = reverse_course_url('checklists_handler', course_key)
            return render_to_response('checklists.html',
                                      {
                                          'handler_url': handler_url,
                                          # context_course is used by analytics
                                          'context_course': course_module,
                                          'checklists': expanded_checklists
                                      })
    elif json_request:
        # Can now assume POST or PUT because GET handled above.
        if checklist_index is not None and 0 <= int(checklist_index) < len(course_module.checklists):
            index = int(checklist_index)
            persisted_checklist = course_module.checklists[index]
            modified_checklist = json.loads(request.body)
            # Only thing the user can modify is the "checked" state.
            # We don't want to persist what comes back from the client because it will
            # include the expanded action URLs (which are non-portable).
            for item_index, item in enumerate(modified_checklist.get('items')):
                persisted_checklist['items'][item_index]['is_checked'] = item['is_checked']
            # seeming noop which triggers kvs to record that the metadata is
            # not default
            course_module.checklists = course_module.checklists
            course_module.save()
            get_modulestore(course_module.location).update_item(course_module, request.user.id)
            expanded_checklist = expand_checklist_action_url(course_module, persisted_checklist)
            return JsonResponse(localize_checklist_text(expanded_checklist))
        else:
            return HttpResponseBadRequest(
                ("Could not save checklist state because the checklist index "
                 "was out of range or unspecified."),
                content_type="text/plain"
            )
    else:
        return HttpResponseNotFound()


def expand_all_action_urls(course_module):
    """
    Gets the checklists out of the course module and expands their action urls.

    Returns a copy of the checklists with modified urls, without modifying the persisted version
    of the checklists.
    """
    expanded_checklists = []
    for checklist in course_module.checklists:
        expanded_checklists.append(localize_checklist_text(expand_checklist_action_url(course_module, checklist)))
    return expanded_checklists


def expand_checklist_action_url(course_module, checklist):
    """
    Expands the action URLs for a given checklist and returns the modified version.

    The method does a copy of the input checklist and does not modify the input argument.
    """
    expanded_checklist = copy.deepcopy(checklist)

    urlconf_map = {
        "ManageUsers": "course_team_handler",
        "CourseOutline": "course_handler",
        "SettingsDetails": "settings_handler",
        "SettingsGrading": "grading_handler",
    }

    for item in expanded_checklist.get('items'):
        action_url = item.get('action_url')
        if action_url in urlconf_map:
            item['action_url'] = reverse_course_url(urlconf_map[action_url], course_module.id)

    return expanded_checklist

def localize_checklist_text(checklist):
    """
    Localize texts for a given checklist and returns the modified version.

    The method does an in-place operation so the input checklist is modified directly.
    """
    # Localize checklist name
    checklist['short_description'] = ugettext(checklist['short_description'])

    # Localize checklist items
    for item in checklist.get('items'):
        item['short_description'] = ugettext(item['short_description'])
        item['long_description'] = ugettext(item['long_description'])
        item['action_text'] = ugettext(item['action_text']) if item['action_text'] != "" else u""

    return checklist

########NEW FILE########
__FILENAME__ = component
from __future__ import absolute_import

import json
import logging

from django.http import HttpResponseBadRequest, Http404
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_GET
from django.core.exceptions import PermissionDenied
from django.conf import settings
from xmodule.modulestore.exceptions import ItemNotFoundError
from edxmako.shortcuts import render_to_response

from util.date_utils import get_default_time_display
from xmodule.modulestore.django import modulestore

from xblock.core import XBlock
from xblock.django.request import webob_to_django_response, django_to_webob_request
from xblock.exceptions import NoSuchHandlerError
from xblock.fields import Scope
from xblock.plugin import PluginMissingError
from xblock.runtime import Mixologist

from contentstore.utils import get_lms_link_for_item, compute_publish_state, PublishState, get_modulestore
from contentstore.views.helpers import get_parent_xblock

from models.settings.course_grading import CourseGradingModel
from xmodule.modulestore.keys import UsageKey

from .access import has_course_access
from django.utils.translation import ugettext as _

__all__ = ['OPEN_ENDED_COMPONENT_TYPES',
           'ADVANCED_COMPONENT_POLICY_KEY',
           'subsection_handler',
           'unit_handler',
           'container_handler',
           'component_handler'
           ]

log = logging.getLogger(__name__)

# NOTE: unit_handler assumes this list is disjoint from ADVANCED_COMPONENT_TYPES
COMPONENT_TYPES = ['discussion', 'html', 'problem', 'video']

OPEN_ENDED_COMPONENT_TYPES = ["combinedopenended", "peergrading"]
NOTE_COMPONENT_TYPES = ['notes']

if settings.FEATURES.get('ALLOW_ALL_ADVANCED_COMPONENTS'):
    ADVANCED_COMPONENT_TYPES = sorted(set(name for name, class_ in XBlock.load_classes()) - set(COMPONENT_TYPES))
else:

    ADVANCED_COMPONENT_TYPES = [
        'annotatable',
        'textannotation',  # module for annotating text (with annotation table)
        'videoannotation',  # module for annotating video (with annotation table)
        'imageannotation',  # module for annotating image (with annotation table)
        'word_cloud',
        'graphical_slider_tool',
        'lti',
        'concept',
        'openassessment',  # edx-ora2
    ] + OPEN_ENDED_COMPONENT_TYPES + NOTE_COMPONENT_TYPES

ADVANCED_COMPONENT_CATEGORY = 'advanced'
ADVANCED_COMPONENT_POLICY_KEY = 'advanced_modules'


@require_GET
@login_required
def subsection_handler(request, usage_key_string):
    """
    The restful handler for subsection-specific requests.

    GET
        html: return html page for editing a subsection
        json: not currently supported
    """
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
        usage_key = UsageKey.from_string(usage_key_string)
        try:
            course, item, lms_link = _get_item_in_course(request, usage_key)
        except ItemNotFoundError:
            return HttpResponseBadRequest()

        preview_link = get_lms_link_for_item(usage_key, preview=True)

        # make sure that location references a 'sequential', otherwise return
        # BadRequest
        if item.location.category != 'sequential':
            return HttpResponseBadRequest()

        parent = get_parent_xblock(item)

        # remove all metadata from the generic dictionary that is presented in a
        # more normalized UI. We only want to display the XBlocks fields, not
        # the fields from any mixins that have been added
        fields = getattr(item, 'unmixed_class', item.__class__).fields

        policy_metadata = dict(
            (field.name, field.read_from(item))
            for field
            in fields.values()
            if field.name not in ['display_name', 'start', 'due', 'format'] and field.scope == Scope.settings
        )

        can_view_live = False
        subsection_units = item.get_children()
        for unit in subsection_units:
            state = compute_publish_state(unit)
            if state in (PublishState.public, PublishState.draft):
                can_view_live = True
                break

        return render_to_response(
            'edit_subsection.html',
            {
                'subsection': item,
                'context_course': course,
                'new_unit_category': 'vertical',
                'lms_link': lms_link,
                'preview_link': preview_link,
                'course_graders': json.dumps(CourseGradingModel.fetch(usage_key.course_key).graders),
                'parent_item': parent,
                'locator': usage_key,
                'policy_metadata': policy_metadata,
                'subsection_units': subsection_units,
                'can_view_live': can_view_live
            }
        )
    else:
        return HttpResponseBadRequest("Only supports html requests")


def _load_mixed_class(category):
    """
    Load an XBlock by category name, and apply all defined mixins
    """
    component_class = XBlock.load_class(category, select=settings.XBLOCK_SELECT_FUNCTION)
    mixologist = Mixologist(settings.XBLOCK_MIXINS)
    return mixologist.mix(component_class)


@require_GET
@login_required
def unit_handler(request, usage_key_string):
    """
    The restful handler for unit-specific requests.

    GET
        html: return html page for editing a unit
        json: not currently supported
    """
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
        usage_key = UsageKey.from_string(usage_key_string)
        try:
            course, item, lms_link = _get_item_in_course(request, usage_key)
        except ItemNotFoundError:
            return HttpResponseBadRequest()

        component_templates = _get_component_templates(course)

        xblocks = item.get_children()

        # TODO (cpennington): If we share units between courses,
        # this will need to change to check permissions correctly so as
        # to pick the correct parent subsection
        containing_subsection = get_parent_xblock(item)
        containing_section = get_parent_xblock(containing_subsection)

        # cdodge hack. We're having trouble previewing drafts via jump_to redirect
        # so let's generate the link url here

        # need to figure out where this item is in the list of children as the
        # preview will need this
        index = 1
        for child in containing_subsection.get_children():
            if child.location == item.location:
                break
            index = index + 1

        preview_lms_base = settings.FEATURES.get('PREVIEW_LMS_BASE')

        preview_lms_link = (
            u'//{preview_lms_base}/courses/{org}/{course}/{course_name}/courseware/{section}/{subsection}/{index}'
        ).format(
            preview_lms_base=preview_lms_base,
            lms_base=settings.LMS_BASE,
            org=course.location.org,
            course=course.location.course,
            course_name=course.location.name,
            section=containing_section.location.name,
            subsection=containing_subsection.location.name,
            index=index
        )

        return render_to_response('unit.html', {
            'context_course': course,
            'unit': item,
            'unit_usage_key': usage_key,
            'child_usage_keys': [block.scope_ids.usage_id for block in xblocks],
            'component_templates': json.dumps(component_templates),
            'draft_preview_link': preview_lms_link,
            'published_preview_link': lms_link,
            'subsection': containing_subsection,
            'release_date': (
                get_default_time_display(containing_subsection.start)
                if containing_subsection.start is not None else None
            ),
            'section': containing_section,
            'new_unit_category': 'vertical',
            'unit_state': compute_publish_state(item),
            'published_date': (
                get_default_time_display(item.published_date)
                if item.published_date is not None else None
            ),
        })
    else:
        return HttpResponseBadRequest("Only supports html requests")


# pylint: disable=unused-argument
@require_GET
@login_required
def container_handler(request, usage_key_string):
    """
    The restful handler for container xblock requests.

    GET
        html: returns the HTML page for editing a container
        json: not currently supported
    """
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):

        usage_key = UsageKey.from_string(usage_key_string)
        try:
            course, xblock, __ = _get_item_in_course(request, usage_key)
        except ItemNotFoundError:
            return HttpResponseBadRequest()

        component_templates = _get_component_templates(course)
        ancestor_xblocks = []
        parent = get_parent_xblock(xblock)
        while parent and parent.category != 'sequential':
            ancestor_xblocks.append(parent)
            parent = get_parent_xblock(parent)
        ancestor_xblocks.reverse()

        unit = ancestor_xblocks[0] if ancestor_xblocks else None
        unit_publish_state = compute_publish_state(unit) if unit else None

        return render_to_response('container.html', {
            'context_course': course,  # Needed only for display of menus at top of page.
            'xblock': xblock,
            'unit_publish_state': unit_publish_state,
            'xblock_locator': usage_key,
            'unit': None if not ancestor_xblocks else ancestor_xblocks[0],
            'ancestor_xblocks': ancestor_xblocks,
            'component_templates': json.dumps(component_templates),
        })
    else:
        return HttpResponseBadRequest("Only supports html requests")


def _get_component_templates(course):
    """
    Returns the applicable component templates that can be used by the specified course.
    """
    def create_template_dict(name, cat, boilerplate_name=None, is_common=False):
        """
        Creates a component template dict.

        Parameters
            display_name: the user-visible name of the component
            category: the type of component (problem, html, etc.)
            boilerplate_name: name of boilerplate for filling in default values. May be None.
            is_common: True if "common" problem, False if "advanced". May be None, as it is only used for problems.

        """
        return {
            "display_name": name,
            "category": cat,
            "boilerplate_name": boilerplate_name,
            "is_common": is_common
        }

    component_display_names = {
        'discussion': _("Discussion"),
        'html': _("HTML"),
        'problem': _("Problem"),
        'video': _("Video")
    }
    advanced_component_display_names = {}

    component_templates = []
    # The component_templates array is in the order of "advanced" (if present), followed
    # by the components in the order listed in COMPONENT_TYPES.
    for category in COMPONENT_TYPES:
        templates_for_category = []
        component_class = _load_mixed_class(category)
        # add the default template with localized display name
        # TODO: Once mixins are defined per-application, rather than per-runtime,
        # this should use a cms mixed-in class. (cpennington)
        if hasattr(component_class, 'display_name'):
            display_name = _(component_class.display_name.default) if component_class.display_name.default else _('Blank')
        else:
            display_name = _('Blank')
        templates_for_category.append(create_template_dict(display_name, category))

        # add boilerplates
        if hasattr(component_class, 'templates'):
            for template in component_class.templates():
                filter_templates = getattr(component_class, 'filter_templates', None)
                if not filter_templates or filter_templates(template, course):
                    templates_for_category.append(
                        create_template_dict(
                            _(template['metadata'].get('display_name')),
                            category,
                            template.get('template_id'),
                            template['metadata'].get('markdown') is not None
                        )
                    )
        component_templates.append({
            "type": category,
            "templates": templates_for_category,
            "display_name": component_display_names[category]
            })

    # Check if there are any advanced modules specified in the course policy.
    # These modules should be specified as a list of strings, where the strings
    # are the names of the modules in ADVANCED_COMPONENT_TYPES that should be
    # enabled for the course.
    course_advanced_keys = course.advanced_modules
    advanced_component_templates = {"type": "advanced", "templates": [], "display_name": _("Advanced")}
    # Set component types according to course policy file
    if isinstance(course_advanced_keys, list):
        for category in course_advanced_keys:
            if category in ADVANCED_COMPONENT_TYPES:
                # boilerplates not supported for advanced components
                try:
                    component_class = _load_mixed_class(category)

                    if component_class.display_name.default:
                        template_display_name = _(component_class.display_name.default)
                    else:
                        template_display_name = advanced_component_display_names.get(category, category)
                    advanced_component_templates['templates'].append(
                        create_template_dict(
                            template_display_name,
                            category
                        )
                    )
                except PluginMissingError:
                    # dhm: I got this once but it can happen any time the
                    # course author configures an advanced component which does
                    # not exist on the server. This code here merely
                    # prevents any authors from trying to instantiate the
                    # non-existent component type by not showing it in the menu
                    log.warning(
                        "Advanced component %s does not exist. It will not be added to the Studio new component menu.",
                        category
                    )
                    pass
    else:
        log.error(
            "Improper format for course advanced keys! %s",
            course_advanced_keys
        )
    if len(advanced_component_templates['templates']) > 0:
        component_templates.insert(0, advanced_component_templates)

    return component_templates


@login_required
def _get_item_in_course(request, usage_key):
    """
    Helper method for getting the old location, containing course,
    item, and lms_link for a given locator.

    Verifies that the caller has permission to access this item.
    """
    course_key = usage_key.course_key

    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course = modulestore().get_course(course_key)
    item = get_modulestore(usage_key).get_item(usage_key, depth=1)
    lms_link = get_lms_link_for_item(usage_key)

    return course, item, lms_link


@login_required
def component_handler(request, usage_key_string, handler, suffix=''):
    """
    Dispatch an AJAX action to an xblock

    Args:
        usage_id: The usage-id of the block to dispatch to
        handler (str): The handler to execute
        suffix (str): The remainder of the url to be passed to the handler

    Returns:
        :class:`django.http.HttpResponse`: The response from the handler, converted to a
            django response
    """

    usage_key = UsageKey.from_string(usage_key_string)

    descriptor = get_modulestore(usage_key).get_item(usage_key)
    # Let the module handle the AJAX
    req = django_to_webob_request(request)

    try:
        resp = descriptor.handle(handler, req, suffix)

    except NoSuchHandlerError:
        log.info("XBlock %s attempted to access missing handler %r", descriptor, handler, exc_info=True)
        raise Http404

    # unintentional update to handle any side effects of handle call; so, request user didn't author
    # the change
    get_modulestore(usage_key).update_item(descriptor, None)

    return webob_to_django_response(resp)

########NEW FILE########
__FILENAME__ = course
"""
Views related to operations on course objects
"""
import json
import random
import string  # pylint: disable=W0402

from django.utils.translation import ugettext as _
from django.contrib.auth.decorators import login_required
from django_future.csrf import ensure_csrf_cookie
from django.conf import settings
from django.views.decorators.http import require_http_methods
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponseNotFound
from util.json_request import JsonResponse
from edxmako.shortcuts import render_to_response

from xmodule.error_module import ErrorDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.content import StaticContent
from xmodule.tabs import PDFTextbookTabs

from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import Location, SlashSeparatedCourseKey

from contentstore.course_info_model import get_course_updates, update_course_updates, delete_course_update
from contentstore.utils import (
    get_lms_link_for_item,
    add_extra_panel_tab,
    remove_extra_panel_tab,
    get_modulestore,
    reverse_course_url
)
from models.settings.course_details import CourseDetails, CourseSettingsEncoder

from models.settings.course_grading import CourseGradingModel
from models.settings.course_metadata import CourseMetadata
from util.json_request import expect_json
from util.string_utils import _has_non_ascii_characters

from .access import has_course_access
from .component import (
    OPEN_ENDED_COMPONENT_TYPES,
    NOTE_COMPONENT_TYPES,
    ADVANCED_COMPONENT_POLICY_KEY
)

from django_comment_common.models import assign_default_role
from django_comment_common.utils import seed_permissions_roles

from student.models import CourseEnrollment
from student.roles import CourseRole, UserBasedRole

from xmodule.modulestore.keys import CourseKey
from course_creators.views import get_course_creator_status, add_user_with_status_unrequested
from contentstore import utils
from student.roles import CourseInstructorRole, CourseStaffRole, CourseCreatorRole, GlobalStaff
from student import auth

from microsite_configuration import microsite

__all__ = ['course_info_handler', 'course_handler', 'course_info_update_handler',
           'settings_handler',
           'grading_handler',
           'advanced_settings_handler',
           'textbooks_list_handler', 'textbooks_detail_handler']


def _get_course_module(course_key, user, depth=0):
    """
    Internal method used to calculate and return the locator and course module
    for the view functions in this file.
    """
    if not has_course_access(user, course_key):
        raise PermissionDenied()
    course_module = modulestore().get_course(course_key, depth=depth)
    return course_module


# pylint: disable=unused-argument
@login_required
def course_handler(request, course_key_string=None):
    """
    The restful handler for course specific requests.
    It provides the course tree with the necessary information for identifying and labeling the parts. The root
    will typically be a 'course' object but may not be especially as we support modules.

    GET
        html: return course listing page if not given a course id
        html: return html page overview for the given course if given a course id
        json: return json representing the course branch's index entry as well as dag w/ all of the children
        replaced w/ json docs where each doc has {'_id': , 'display_name': , 'children': }
    POST
        json: create a course, return resulting json
        descriptor (same as in GET course/...). Leaving off /branch/draft would imply create the course w/ default
        branches. Cannot change the structure contents ('_id', 'display_name', 'children') but can change the
        index entry.
    PUT
        json: update this course (index entry not xblock) such as repointing head, changing display name, org,
        offering. Return same json as above.
    DELETE
        json: delete this branch from this course (leaving off /branch/draft would imply delete the course)
    """
    response_format = request.REQUEST.get('format', 'html')
    if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            return JsonResponse(_course_json(request, CourseKey.from_string(course_key_string)))
        elif request.method == 'POST':  # not sure if this is only post. If one will have ids, it goes after access
            return create_new_course(request)
        elif not has_course_access(request.user, CourseKey.from_string(course_key_string)):
            raise PermissionDenied()
        elif request.method == 'PUT':
            raise NotImplementedError()
        elif request.method == 'DELETE':
            raise NotImplementedError()
        else:
            return HttpResponseBadRequest()
    elif request.method == 'GET':  # assume html
        if course_key_string is None:
            return course_listing(request)
        else:
            return course_index(request, CourseKey.from_string(course_key_string))
    else:
        return HttpResponseNotFound()


@login_required
def _course_json(request, course_key):
    """
    Returns a JSON overview of a course
    """
    course_module = _get_course_module(course_key, request.user, depth=None)
    return _xmodule_json(course_module, course_module.id)


def _xmodule_json(xmodule, course_id):
    """
    Returns a JSON overview of an XModule
    """
    is_container = xmodule.has_children
    result = {
        'display_name': xmodule.display_name,
        'id': unicode(xmodule.location),
        'category': xmodule.category,
        'is_draft': getattr(xmodule, 'is_draft', False),
        'is_container': is_container,
    }
    if is_container:
        result['children'] = [_xmodule_json(child, course_id) for child in xmodule.get_children()]
    return result


def _accessible_courses_list(request):
    """
    List all courses available to the logged in user by iterating through all the courses
    """
    courses = modulestore('direct').get_courses()

    # filter out courses that we don't have access to
    def course_filter(course):
        """
        Get courses to which this user has access
        """
        if GlobalStaff().has_user(request.user):
            return course.location.course != 'templates'

        return (has_course_access(request.user, course.id)
                # pylint: disable=fixme
                # TODO remove this condition when templates purged from db
                and course.location.course != 'templates'
                )
    courses = filter(course_filter, courses)
    return courses


def _accessible_courses_list_from_groups(request):
    """
    List all courses available to the logged in user by reversing access group names
    """
    courses_list = {}

    instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role()
    staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role()
    all_courses = instructor_courses | staff_courses

    for course_access in all_courses:
        course_key = course_access.course_id
        if course_key not in courses_list:
            course = modulestore('direct').get_course(course_key)
            if course is None:
                raise ItemNotFoundError(course_key)
            courses_list[course_key] = course

    return courses_list.values()


@login_required
@ensure_csrf_cookie
def course_listing(request):
    """
    List all courses available to the logged in user
    Try to get all courses by first reversing django groups and fallback to old method if it fails
    Note: overhead of pymongo reads will increase if getting courses from django groups fails
    """
    if GlobalStaff().has_user(request.user):
        # user has global access so no need to get courses from django groups
        courses = _accessible_courses_list(request)
    else:
        try:
            courses = _accessible_courses_list_from_groups(request)
        except ItemNotFoundError:
            # user have some old groups or there was some error getting courses from django groups
            # so fallback to iterating through all courses
            courses = _accessible_courses_list(request)

    def format_course_for_view(course):
        """
        return tuple of the data which the view requires for each course
        """
        return (
            course.display_name,
            reverse_course_url('course_handler', course.id),
            get_lms_link_for_item(course.location),
            course.display_org_with_default,
            course.display_number_with_default,
            course.location.name
        )

    return render_to_response('index.html', {
        'courses': [format_course_for_view(c) for c in courses if not isinstance(c, ErrorDescriptor)],
        'user': request.user,
        'request_course_creator_url': reverse('contentstore.views.request_course_creator'),
        'course_creator_status': _get_course_creator_status(request.user),
        'allow_unicode_course_id': settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID', False)
    })


@login_required
@ensure_csrf_cookie
def course_index(request, course_key):
    """
    Display an editable course overview.

    org, course, name: Attributes of the Location for the item to edit
    """
    course_module = _get_course_module(course_key, request.user, depth=3)
    lms_link = get_lms_link_for_item(course_module.location)
    sections = course_module.get_children()


    return render_to_response('overview.html', {
        'context_course': course_module,
        'lms_link': lms_link,
        'sections': sections,
        'course_graders': json.dumps(
            CourseGradingModel.fetch(course_key).graders
        ),
        'new_section_category': 'chapter',
        'new_subsection_category': 'sequential',
        'new_unit_category': 'vertical',
        'category': 'vertical'
    })


@expect_json
def create_new_course(request):
    """
    Create a new course.

    Returns the URL for the course overview page.
    """
    if not auth.has_access(request.user, CourseCreatorRole()):
        raise PermissionDenied()

    org = request.json.get('org')
    number = request.json.get('number')
    display_name = request.json.get('display_name')
    run = request.json.get('run')

    # allow/disable unicode characters in course_id according to settings
    if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'):
        if _has_non_ascii_characters(org) or _has_non_ascii_characters(number) or _has_non_ascii_characters(run):
            return JsonResponse(
                {'error': _('Special characters not allowed in organization, course number, and course run.')},
                status=400
            )

    try:
        course_key = SlashSeparatedCourseKey(org, number, run)

        # instantiate the CourseDescriptor and then persist it
        # note: no system to pass
        if display_name is None:
            metadata = {}
        else:
            metadata = {'display_name': display_name}

        # Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for
        # existing xml courses this cannot be changed in CourseDescriptor.
        # # TODO get rid of defining wiki slug in this org/course/run specific way and reconcile
        # w/ xmodule.course_module.CourseDescriptor.__init__
        wiki_slug = u"{0}.{1}.{2}".format(course_key.org, course_key.course, course_key.run)
        definition_data = {'wiki_slug': wiki_slug}

        # Create the course then fetch it from the modulestore
        # Check if role permissions group for a course named like this already exists
        # Important because role groups are case insensitive
        if CourseRole.course_group_already_exists(course_key):
            raise InvalidLocationError()

        fields = {}
        fields.update(definition_data)
        fields.update(metadata)

        # Creating the course raises InvalidLocationError if an existing course with this org/name is found
        new_course = modulestore('direct').create_course(
            course_key.org,
            course_key.offering,
            fields=fields,
        )

        # can't use auth.add_users here b/c it requires request.user to already have Instructor perms in this course
        # however, we can assume that b/c this user had authority to create the course, the user can add themselves
        CourseInstructorRole(new_course.id).add_users(request.user)
        auth.add_users(request.user, CourseStaffRole(new_course.id), request.user)

        # seed the forums
        seed_permissions_roles(new_course.id)

        # auto-enroll the course creator in the course so that "View Live" will
        # work.
        CourseEnrollment.enroll(request.user, new_course.id)
        _users_assign_default_role(new_course.id)

        return JsonResponse({
            'url': reverse_course_url('course_handler', new_course.id)
        })

    except InvalidLocationError:
        return JsonResponse({
            'ErrMsg': _(
                'There is already a course defined with the same '
                'organization, course number, and course run. Please '
                'change either organization or course number to be unique.'
            ),
            'OrgErrMsg': _(
                'Please change either the organization or '
                'course number so that it is unique.'),
            'CourseErrMsg': _(
                'Please change either the organization or '
                'course number so that it is unique.'),
        })
    except InvalidKeyError as error:
        return JsonResponse({
            "ErrMsg": _("Unable to create course '{name}'.\n\n{err}").format(name=display_name, err=error.message)}
        )


def _users_assign_default_role(course_id):
    """
    Assign 'Student' role to all previous users (if any) for this course
    """
    enrollments = CourseEnrollment.objects.filter(course_id=course_id)
    for enrollment in enrollments:
        assign_default_role(course_id, enrollment.user)


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(["GET"])
def course_info_handler(request, course_key_string):
    """
    GET
        html: return html for editing the course info handouts and updates.
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):

        return render_to_response(
            'course_info.html',
            {
                'context_course': course_module,
                'updates_url': reverse_course_url('course_info_update_handler', course_key),
                'handouts_locator': course_key.make_usage_key('course_info', 'handouts'),
                'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.id)
            }
        )
    else:
        return HttpResponseBadRequest("Only supports html requests")


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
@expect_json
def course_info_update_handler(request, course_key_string, provided_id=None):
    """
    restful CRUD operations on course_info updates.
    provided_id should be none if it's new (create) and index otherwise.
    GET
        json: return the course info update models
    POST
        json: create an update
    PUT or DELETE
        json: change an existing update
    """
    if 'application/json' not in request.META.get('HTTP_ACCEPT', 'application/json'):
        return HttpResponseBadRequest("Only supports json requests")

    course_key = CourseKey.from_string(course_key_string)
    usage_key = course_key.make_usage_key('course_info', 'updates')
    if provided_id == '':
        provided_id = None

    # check that logged in user has permissions to this item (GET shouldn't require this level?)
    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    if request.method == 'GET':
        course_updates = get_course_updates(usage_key, provided_id)
        if isinstance(course_updates, dict) and course_updates.get('error'):
            return JsonResponse(course_updates, course_updates.get('status', 400))
        else:
            return JsonResponse(course_updates)
    elif request.method == 'DELETE':
        try:
            return JsonResponse(delete_course_update(usage_key, request.json, provided_id, request.user))
        except:
            return HttpResponseBadRequest(
                "Failed to delete",
                content_type="text/plain"
            )
    # can be either and sometimes django is rewriting one to the other:
    elif request.method in ('POST', 'PUT'):
        try:
            return JsonResponse(update_course_updates(usage_key, request.json, provided_id, request.user))
        except:
            return HttpResponseBadRequest(
                "Failed to save",
                content_type="text/plain"
            )


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "PUT", "POST"))
@expect_json
def settings_handler(request, course_key_string):
    """
    Course settings for dates and about pages
    GET
        html: get the page
        json: get the CourseDetails model
    PUT
        json: update the Course and About xblocks through the CourseDetails model
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
        upload_asset_url = reverse_course_url('assets_handler', course_key)

        # see if the ORG of this course can be attributed to a 'Microsite'. In that case, the
        # course about page should be editable in Studio
        about_page_editable = not microsite.get_value_for_org(
            course_module.location.org,
            'ENABLE_MKTG_SITE',
            settings.FEATURES.get('ENABLE_MKTG_SITE', False)
        )

        short_description_editable = settings.FEATURES.get('EDITABLE_SHORT_DESCRIPTION', True)

        return render_to_response('settings.html', {
            'context_course': course_module,
            'course_locator': course_key,
            'lms_link_for_about_page': utils.get_lms_link_for_about_page(course_key),
            'course_image_url': utils.course_image_url(course_module),
            'details_url': reverse_course_url('settings_handler', course_key),
            'about_page_editable': about_page_editable,
            'short_description_editable': short_description_editable,
            'upload_asset_url': upload_asset_url
        })
    elif 'application/json' in request.META.get('HTTP_ACCEPT', ''):
        if request.method == 'GET':
            return JsonResponse(
                CourseDetails.fetch(course_key),
                # encoder serializes dates, old locations, and instances
                encoder=CourseSettingsEncoder
            )
        else:  # post or put, doesn't matter.
            return JsonResponse(
                CourseDetails.update_from_json(course_key, request.json, request.user),
                encoder=CourseSettingsEncoder
            )


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
@expect_json
def grading_handler(request, course_key_string, grader_index=None):
    """
    Course Grading policy configuration
    GET
        html: get the page
        json no grader_index: get the CourseGrading model (graceperiod, cutoffs, and graders)
        json w/ grader_index: get the specific grader
    PUT
        json no grader_index: update the Course through the CourseGrading model
        json w/ grader_index: create or update the specific grader (create if index out of range)
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)

    if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
        course_details = CourseGradingModel.fetch(course_key)

        return render_to_response('settings_graders.html', {
            'context_course': course_module,
            'course_locator': course_key,
            'course_details': json.dumps(course_details, cls=CourseSettingsEncoder),
            'grading_url': reverse_course_url('grading_handler', course_key),
        })
    elif 'application/json' in request.META.get('HTTP_ACCEPT', ''):
        if request.method == 'GET':
            if grader_index is None:
                return JsonResponse(
                    CourseGradingModel.fetch(course_key),
                    # encoder serializes dates, old locations, and instances
                    encoder=CourseSettingsEncoder
                )
            else:
                return JsonResponse(CourseGradingModel.fetch_grader(course_key, grader_index))
        elif request.method in ('POST', 'PUT'):  # post or put, doesn't matter.
            # None implies update the whole model (cutoffs, graceperiod, and graders) not a specific grader
            if grader_index is None:
                return JsonResponse(
                    CourseGradingModel.update_from_json(course_key, request.json, request.user),
                    encoder=CourseSettingsEncoder
                )
            else:
                return JsonResponse(
                    CourseGradingModel.update_grader_from_json(course_key, request.json, request.user)
                )
        elif request.method == "DELETE" and grader_index is not None:
            CourseGradingModel.delete_grader(course_key, grader_index, request.user)
            return JsonResponse()


# pylint: disable=invalid-name
def _config_course_advanced_components(request, course_module):
    """
    Check to see if the user instantiated any advanced components. This
    is a hack that does the following :
    1) adds/removes the open ended panel tab to a course automatically
    if the user has indicated that they want to edit the
    combinedopendended or peergrading module
    2) adds/removes the notes panel tab to a course automatically if
    the user has indicated that they want the notes module enabled in
    their course
    """
    # TODO refactor the above into distinct advanced policy settings
    filter_tabs = True  # Exceptional conditions will pull this to False
    if ADVANCED_COMPONENT_POLICY_KEY in request.json:  # Maps tab types to components
        tab_component_map = {
            'open_ended': OPEN_ENDED_COMPONENT_TYPES,
            'notes': NOTE_COMPONENT_TYPES,
        }
        # Check to see if the user instantiated any notes or open ended components
        for tab_type in tab_component_map.keys():
            component_types = tab_component_map.get(tab_type)
            found_ac_type = False
            for ac_type in component_types:
                if ac_type in request.json[ADVANCED_COMPONENT_POLICY_KEY]:
                    # Add tab to the course if needed
                    changed, new_tabs = add_extra_panel_tab(tab_type, course_module)
                    # If a tab has been added to the course, then send the
                    # metadata along to CourseMetadata.update_from_json
                    if changed:
                        course_module.tabs = new_tabs
                        request.json.update({'tabs': new_tabs})
                        # Indicate that tabs should not be filtered out of
                        # the metadata
                        filter_tabs = False  # Set this flag to avoid the tab removal code below.
                    found_ac_type = True  #break

            # If we did not find a module type in the advanced settings,
            # we may need to remove the tab from the course.
            if not found_ac_type:  # Remove tab from the course if needed
                changed, new_tabs = remove_extra_panel_tab(tab_type, course_module)
                if changed:
                    course_module.tabs = new_tabs
                    request.json.update({'tabs':new_tabs})
                    # Indicate that tabs should *not* be filtered out of
                    # the metadata
                    filter_tabs = False

    return filter_tabs


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
@expect_json
def advanced_settings_handler(request, course_key_string):
    """
    Course settings configuration
    GET
        html: get the page
        json: get the model
    PUT, POST
        json: update the Course's settings. The payload is a json rep of the
            metadata dicts. The dict can include a "unsetKeys" entry which is a list
            of keys whose values to unset: i.e., revert to default
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':

        return render_to_response('settings_advanced.html', {
            'context_course': course_module,
            'advanced_dict': json.dumps(CourseMetadata.fetch(course_module)),
            'advanced_settings_url': reverse_course_url('advanced_settings_handler', course_key)
        })
    elif 'application/json' in request.META.get('HTTP_ACCEPT', ''):
        if request.method == 'GET':
            return JsonResponse(CourseMetadata.fetch(course_module))
        else:
            # Whether or not to filter the tabs key out of the settings metadata
            filter_tabs = _config_course_advanced_components(request, course_module)
            try:
                return JsonResponse(CourseMetadata.update_from_json(
                    course_module,
                    request.json,
                    filter_tabs=filter_tabs,
                    user=request.user,
                ))
            except (TypeError, ValueError) as err:
                return HttpResponseBadRequest(
                    "Incorrect setting format. {}".format(err),
                    content_type="text/plain"
                )


class TextbookValidationError(Exception):
    "An error thrown when a textbook input is invalid"
    pass


def validate_textbooks_json(text):
    """
    Validate the given text as representing a single PDF textbook
    """
    try:
        textbooks = json.loads(text)
    except ValueError:
        raise TextbookValidationError("invalid JSON")
    if not isinstance(textbooks, (list, tuple)):
        raise TextbookValidationError("must be JSON list")
    for textbook in textbooks:
        validate_textbook_json(textbook)
    # check specified IDs for uniqueness
    all_ids = [textbook["id"] for textbook in textbooks if "id" in textbook]
    unique_ids = set(all_ids)
    if len(all_ids) > len(unique_ids):
        raise TextbookValidationError("IDs must be unique")
    return textbooks


def validate_textbook_json(textbook):
    """
    Validate the given text as representing a list of PDF textbooks
    """
    if isinstance(textbook, basestring):
        try:
            textbook = json.loads(textbook)
        except ValueError:
            raise TextbookValidationError("invalid JSON")
    if not isinstance(textbook, dict):
        raise TextbookValidationError("must be JSON object")
    if not textbook.get("tab_title"):
        raise TextbookValidationError("must have tab_title")
    tid = unicode(textbook.get("id", ""))
    if tid and not tid[0].isdigit():
        raise TextbookValidationError("textbook ID must start with a digit")
    return textbook


def assign_textbook_id(textbook, used_ids=()):
    """
    Return an ID that can be assigned to a textbook
    and doesn't match the used_ids
    """
    tid = Location.clean(textbook["tab_title"])
    if not tid[0].isdigit():
        # stick a random digit in front
        tid = random.choice(string.digits) + tid
    while tid in used_ids:
        # add a random ASCII character to the end
        tid = tid + random.choice(string.ascii_lowercase)
    return tid


@require_http_methods(("GET", "POST", "PUT"))
@login_required
@ensure_csrf_cookie
def textbooks_list_handler(request, course_key_string):
    """
    A RESTful handler for textbook collections.

    GET
        html: return textbook list page (Backbone application)
        json: return JSON representation of all textbooks in this course
    POST
        json: create a new textbook for this course
    PUT
        json: overwrite all textbooks in the course with the given list
    """
    course_key = CourseKey.from_string(course_key_string)
    course = _get_course_module(course_key, request.user)
    store = get_modulestore(course.location)

    if not "application/json" in request.META.get('HTTP_ACCEPT', 'text/html'):
        # return HTML page
        upload_asset_url = reverse_course_url('assets_handler', course_key)
        textbook_url = reverse_course_url('textbooks_list_handler', course_key)
        return render_to_response('textbooks.html', {
            'context_course': course,
            'textbooks': course.pdf_textbooks,
            'upload_asset_url': upload_asset_url,
            'textbook_url': textbook_url,
        })

    # from here on down, we know the client has requested JSON
    if request.method == 'GET':
        return JsonResponse(course.pdf_textbooks)
    elif request.method == 'PUT':
        try:
            textbooks = validate_textbooks_json(request.body)
        except TextbookValidationError as err:
            return JsonResponse({"error": err.message}, status=400)

        tids = set(t["id"] for t in textbooks if "id" in t)
        for textbook in textbooks:
            if not "id" in textbook:
                tid = assign_textbook_id(textbook, tids)
                textbook["id"] = tid
                tids.add(tid)

        if not any(tab['type'] == PDFTextbookTabs.type for tab in course.tabs):
            course.tabs.append(PDFTextbookTabs())
        course.pdf_textbooks = textbooks
        store.update_item(course, request.user.id)
        return JsonResponse(course.pdf_textbooks)
    elif request.method == 'POST':
        # create a new textbook for the course
        try:
            textbook = validate_textbook_json(request.body)
        except TextbookValidationError as err:
            return JsonResponse({"error": err.message}, status=400)
        if not textbook.get("id"):
            tids = set(t["id"] for t in course.pdf_textbooks if "id" in t)
            textbook["id"] = assign_textbook_id(textbook, tids)
        existing = course.pdf_textbooks
        existing.append(textbook)
        course.pdf_textbooks = existing
        if not any(tab['type'] == PDFTextbookTabs.type for tab in course.tabs):
            course.tabs.append(PDFTextbookTabs())
        store.update_item(course, request.user.id)
        resp = JsonResponse(textbook, status=201)
        resp["Location"] = reverse_course_url(
            'textbooks_detail_handler',
            course.id,
            kwargs={'textbook_id': textbook["id"]}
        )
        return resp


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
def textbooks_detail_handler(request, course_key_string, textbook_id):
    """
    JSON API endpoint for manipulating a textbook via its internal ID.
    Used by the Backbone application.

    GET
        json: return JSON representation of textbook
    POST or PUT
        json: update textbook based on provided information
    DELETE
        json: remove textbook
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    store = get_modulestore(course_module.location)
    matching_id = [tb for tb in course_module.pdf_textbooks
                   if unicode(tb.get("id")) == unicode(textbook_id)]
    if matching_id:
        textbook = matching_id[0]
    else:
        textbook = None

    if request.method == 'GET':
        if not textbook:
            return JsonResponse(status=404)
        return JsonResponse(textbook)
    elif request.method in ('POST', 'PUT'):  # can be either and sometimes
                                        # django is rewriting one to the other
        try:
            new_textbook = validate_textbook_json(request.body)
        except TextbookValidationError as err:
            return JsonResponse({"error": err.message}, status=400)
        new_textbook["id"] = textbook_id
        if textbook:
            i = course_module.pdf_textbooks.index(textbook)
            new_textbooks = course_module.pdf_textbooks[0:i]
            new_textbooks.append(new_textbook)
            new_textbooks.extend(course_module.pdf_textbooks[i + 1:])
            course_module.pdf_textbooks = new_textbooks
        else:
            course_module.pdf_textbooks.append(new_textbook)
        store.update_item(course_module, request.user.id)
        return JsonResponse(new_textbook, status=201)
    elif request.method == 'DELETE':
        if not textbook:
            return JsonResponse(status=404)
        i = course_module.pdf_textbooks.index(textbook)
        remaining_textbooks = course_module.pdf_textbooks[0:i]
        remaining_textbooks.extend(course_module.pdf_textbooks[i + 1:])
        course_module.pdf_textbooks = remaining_textbooks
        store.update_item(course_module, request.user.id)
        return JsonResponse()


def _get_course_creator_status(user):
    """
    Helper method for returning the course creator status for a particular user,
    taking into account the values of DISABLE_COURSE_CREATION and ENABLE_CREATOR_GROUP.

    If the user passed in has not previously visited the index page, it will be
    added with status 'unrequested' if the course creator group is in use.
    """
    if user.is_staff:
        course_creator_status = 'granted'
    elif settings.FEATURES.get('DISABLE_COURSE_CREATION', False):
        course_creator_status = 'disallowed_for_this_site'
    elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False):
        course_creator_status = get_course_creator_status(user)
        if course_creator_status is None:
            # User not grandfathered in as an existing user, has not previously visited the dashboard page.
            # Add the user to the course creator admin table with status 'unrequested'.
            add_user_with_status_unrequested(user)
            course_creator_status = get_course_creator_status(user)
    else:
        course_creator_status = 'granted'

    return course_creator_status

########NEW FILE########
__FILENAME__ = dev
"""
Views that are only activated when the project is running in development mode.
These views will NOT be shown on production: trying to access them will result
in a 404 error.
"""
# pylint: disable=W0613
from edxmako.shortcuts import render_to_response
from mako.exceptions import TopLevelLookupException
from django.http import HttpResponseNotFound


def dev_mode(request):
    "Sample static view"
    return render_to_response("dev/dev_mode.html")


def dev_show_template(request, template):
    """
    Shows the specified template as an HTML page.
    e.g. /template/ux/reference/container.html shows the template under ux/reference/container.html

    Note: dynamic parameters can also be passed to the page.
    e.g. /template/ux/reference/container.html?name=Foo
    """
    try:
        return render_to_response(template, request.GET.dict())
    except TopLevelLookupException:
        return HttpResponseNotFound("Couldn't find template {tpl}".format(tpl=template))

########NEW FILE########
__FILENAME__ = error
# pylint: disable=C0111,W0613

from django.http import (HttpResponse, HttpResponseServerError,
                         HttpResponseNotFound)
from edxmako.shortcuts import render_to_string, render_to_response
import functools
import json

__all__ = ['not_found', 'server_error', 'render_404', 'render_500']


def jsonable_error(status=500, message="The Studio servers encountered an error"):
    """
    A decorator to make an error view return an JSON-formatted message if
    it was requested via AJAX.
    """
    def outer(func):
        @functools.wraps(func)
        def inner(request, *args, **kwargs):
            if request.is_ajax():
                content = json.dumps({"error": message})
                return HttpResponse(content, content_type="application/json",
                                    status=status)
            else:
                return func(request, *args, **kwargs)
        return inner
    return outer


@jsonable_error(404, "Resource not found")
def not_found(request):
    return render_to_response('error.html', {'error': '404'})


@jsonable_error(500, "The Studio servers encountered an error")
def server_error(request):
    return render_to_response('error.html', {'error': '500'})


@jsonable_error(404, "Resource not found")
def render_404(request):
    return HttpResponseNotFound(render_to_string('404.html', {}))


@jsonable_error(500, "The Studio servers encountered an error")
def render_500(request):
    return HttpResponseServerError(render_to_string('500.html', {}))

########NEW FILE########
__FILENAME__ = export_git
"""
This views handles exporting the course xml to a git repository if
the giturl attribute is set.
"""

import logging

from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django_future.csrf import ensure_csrf_cookie
from django.utils.translation import ugettext as _

from .access import has_course_access
import contentstore.git_export_utils as git_export_utils
from edxmako.shortcuts import render_to_response
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey

log = logging.getLogger(__name__)


@ensure_csrf_cookie
@login_required
def export_git(request, course_key_string):
    """
    This method serves up the 'Export to Git' page
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)
    failed = False

    log.debug('export_git course_module=%s', course_module)

    msg = ""
    if 'action' in request.GET and course_module.giturl:
        if request.GET['action'] == 'push':
            try:
                git_export_utils.export_to_git(
                    course_module.id,
                    course_module.giturl,
                    request.user,
                )
                msg = _('Course successfully exported to git repository')
            except git_export_utils.GitExportError as ex:
                failed = True
                msg = unicode(ex)

    return render_to_response('export_git.html', {
        'context_course': course_module,
        'msg': msg,
        'failed': failed,
    })

########NEW FILE########
__FILENAME__ = helpers
import logging

from django.http import HttpResponse
from django.shortcuts import redirect
from edxmako.shortcuts import render_to_string, render_to_response
from xmodule.modulestore.django import modulestore
from contentstore.utils import reverse_course_url, reverse_usage_url

__all__ = ['edge', 'event', 'landing']

EDITING_TEMPLATES = [
    "basic-modal", "modal-button", "edit-xblock-modal", "editor-mode-button", "upload-dialog", "image-modal",
    "add-xblock-component", "add-xblock-component-button", "add-xblock-component-menu",
    "add-xblock-component-menu-problem"
]

# points to the temporary course landing page with log in and sign up
def landing(request, org, course, coursename):
    return render_to_response('temp-course-landing.html', {})


# points to the temporary edge page
def edge(request):
    return redirect('/')


def event(request):
    '''
    A noop to swallow the analytics call so that cms methods don't spook and poor developers looking at
    console logs don't get distracted :-)
    '''
    return HttpResponse(status=204)


def render_from_lms(template_name, dictionary, context=None, namespace='main'):
    """
    Render a template using the LMS MAKO_TEMPLATES
    """
    return render_to_string(template_name, dictionary, context, namespace="lms." + namespace)


def _xmodule_recurse(item, action, ignore_exception=()):
    """
    Recursively apply provided action on item and its children

    ignore_exception (Exception Object): A optional argument; when passed ignores the corresponding
        exception raised during xmodule recursion,
    """
    for child in item.get_children():
        _xmodule_recurse(child, action, ignore_exception)

    try:
        return action(item)
    except ignore_exception:
        return


def get_parent_xblock(xblock):
    """
    Returns the xblock that is the parent of the specified xblock, or None if it has no parent.
    """
    locator = xblock.location
    parent_locations = modulestore().get_parent_locations(locator,)

    if len(parent_locations) == 0:
        return None
    elif len(parent_locations) > 1:
        logging.error('Multiple parents have been found for %s', unicode(locator))
    return modulestore().get_item(parent_locations[0])


def is_unit(xblock):
    """
    Returns true if the specified xblock is a vertical that is treated as a unit.
    A unit is a vertical that is a direct child of a sequential (aka a subsection).
    """
    if xblock.category == 'vertical':
        parent_xblock = get_parent_xblock(xblock)
        parent_category = parent_xblock.category if parent_xblock else None
        return parent_category == 'sequential'
    return False


def xblock_has_own_studio_page(xblock):
    """
    Returns true if the specified xblock has an associated Studio page. Most xblocks do
    not have their own page but are instead shown on the page of their parent. There
    are a few exceptions:
      1. Courses
      2. Verticals that are either:
        - themselves treated as units (in which case they are shown on a unit page)
        - a direct child of a unit (in which case they are shown on a container page)
      3. XBlocks with children, except for:
        - sequentials (aka subsections)
        - chapters (aka sections)
    """
    category = xblock.category

    if is_unit(xblock):
        return True
    elif category == 'vertical':
        parent_xblock = get_parent_xblock(xblock)
        return is_unit(parent_xblock) if parent_xblock else False
    elif category in ('sequential', 'chapter'):
        return False

    # All other xblocks with children have their own page
    return xblock.has_children


def xblock_studio_url(xblock):
    """
    Returns the Studio editing URL for the specified xblock.
    """
    if not xblock_has_own_studio_page(xblock):
        return None
    category = xblock.category
    parent_xblock = get_parent_xblock(xblock)
    parent_category = parent_xblock.category if parent_xblock else None
    if category == 'course':
        return reverse_course_url('course_handler', xblock.location.course_key)
    elif category == 'vertical' and parent_category == 'sequential':
        # only show the unit page for verticals directly beneath a subsection
        return reverse_usage_url('unit_handler', xblock.location)
    else:
        return reverse_usage_url('container_handler', xblock.location)

########NEW FILE########
__FILENAME__ = import_export
"""
These views handle all actions in Studio related to import and exporting of
courses
"""
import logging
import os
import re
import shutil
import tarfile
from path import path
from tempfile import mkdtemp

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.core.files.temp import NamedTemporaryFile
from django.core.servers.basehttp import FileWrapper
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_http_methods, require_GET

from django_future.csrf import ensure_csrf_cookie
from edxmako.shortcuts import render_to_response
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import SerializationError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml

from .access import has_course_access

from extract_tar import safetar_extractall
from student import auth
from student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff
from util.json_request import JsonResponse

from contentstore.utils import reverse_course_url, reverse_usage_url


__all__ = ['import_handler', 'import_status_handler', 'export_handler']


log = logging.getLogger(__name__)


# Regex to capture Content-Range header ranges.
CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})")


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
def import_handler(request, course_key_string):
    """
    The restful handler for importing a course.

    GET
        html: return html page for import page
        json: not supported
    POST or PUT
        json: import a course via the .tar.gz file specified in request.FILES
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()


    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            raise NotImplementedError('coming soon')
        else:
            data_root = path(settings.GITHUB_REPO_ROOT)
            course_subdir = "{0}-{1}-{2}".format(course_key.org, course_key.course, course_key.run)
            course_dir = data_root / course_subdir

            filename = request.FILES['course-data'].name
            if not filename.endswith('.tar.gz'):
                return JsonResponse(
                    {
                        'ErrMsg': _('We only support uploading a .tar.gz file.'),
                        'Stage': 1
                    },
                    status=415
                )
            temp_filepath = course_dir / filename

            if not course_dir.isdir():
                os.mkdir(course_dir)

            logging.debug('importing course to {0}'.format(temp_filepath))

            # Get upload chunks byte ranges
            try:
                matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
                content_range = matches.groupdict()
            except KeyError:    # Single chunk
                # no Content-Range header, so make one that will work
                content_range = {'start': 0, 'stop': 1, 'end': 2}

            # stream out the uploaded files in chunks to disk
            if int(content_range['start']) == 0:
                mode = "wb+"
            else:
                mode = "ab+"
                size = os.path.getsize(temp_filepath)
                # Check to make sure we haven't missed a chunk
                # This shouldn't happen, even if different instances are handling
                # the same session, but it's always better to catch errors earlier.
                if size < int(content_range['start']):
                    log.warning(
                        "Reported range %s does not match size downloaded so far %s",
                        content_range['start'],
                        size
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': _('File upload corrupted. Please try again'),
                            'Stage': 1
                        },
                        status=409
                    )
                # The last request sometimes comes twice. This happens because
                # nginx sends a 499 error code when the response takes too long.
                elif size > int(content_range['stop']) and size == int(content_range['end']):
                    return JsonResponse({'ImportStatus': 1})

            with open(temp_filepath, mode) as temp_file:
                for chunk in request.FILES['course-data'].chunks():
                    temp_file.write(chunk)

            size = os.path.getsize(temp_filepath)

            if int(content_range['stop']) != int(content_range['end']) - 1:
                # More chunks coming
                return JsonResponse({
                    "files": [{
                                  "name": filename,
                                  "size": size,
                                  "deleteUrl": "",
                                  "deleteType": "",
                                  "url": reverse_course_url('import_handler', course_key),
                                  "thumbnailUrl": ""
                              }]
                })

            else:   # This was the last chunk.

                # Use sessions to keep info about import progress
                session_status = request.session.setdefault("import_status", {})
                key = unicode(course_key) + filename
                session_status[key] = 1
                request.session.modified = True

                # Do everything from now on in a try-finally block to make sure
                # everything is properly cleaned up.
                try:

                    tar_file = tarfile.open(temp_filepath)
                    try:
                        safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
                    except SuspiciousOperation as exc:
                        return JsonResponse(
                            {
                                'ErrMsg': 'Unsafe tar file. Aborting import.',
                                'SuspiciousFileOperationMsg': exc.args[0],
                                'Stage': 1
                            },
                            status=400
                        )
                    finally:
                        tar_file.close()

                    session_status[key] = 2
                    request.session.modified = True

                    # find the 'course.xml' file
                    def get_all_files(directory):
                        """
                        For each file in the directory, yield a 2-tuple of (file-name,
                        directory-path)
                        """
                        for dirpath, _dirnames, filenames in os.walk(directory):
                            for filename in filenames:
                                yield (filename, dirpath)

                    def get_dir_for_fname(directory, filename):
                        """
                        Returns the dirpath for the first file found in the directory
                        with the given name.  If there is no file in the directory with
                        the specified name, return None.
                        """
                        for fname, dirpath in get_all_files(directory):
                            if fname == filename:
                                return dirpath
                        return None

                    fname = "course.xml"

                    dirpath = get_dir_for_fname(course_dir, fname)

                    if not dirpath:
                        return JsonResponse(
                            {

                                'ErrMsg': _('Could not find the course.xml file in the package.'),
                                'Stage': 2
                            },
                            status=415
                        )

                    logging.debug('found course.xml at {0}'.format(dirpath))

                    if dirpath != course_dir:
                        for fname in os.listdir(dirpath):
                            shutil.move(dirpath / fname, course_dir)

                    _module_store, course_items = import_from_xml(
                        modulestore('direct'),
                        settings.GITHUB_REPO_ROOT,
                        [course_subdir],
                        load_error_modules=False,
                        static_content_store=contentstore(),
                        target_course_id=course_key,
                        draft_store=modulestore()
                    )

                    new_location = course_items[0].location
                    logging.debug('new course at {0}'.format(new_location))

                    session_status[key] = 3
                    request.session.modified = True

                # Send errors to client with stage at which error occurred.
                except Exception as exception:   # pylint: disable=W0703
                    log.exception(
                        "error importing course"
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': str(exception),
                            'Stage': session_status[key]
                        },
                        status=400
                    )

                finally:
                    shutil.rmtree(course_dir)

                return JsonResponse({'Status': 'OK'})
    elif request.method == 'GET':  # assume html
        course_module = modulestore().get_course(course_key)
        return render_to_response('import.html', {
            'context_course': course_module,
            'successful_import_redirect_url': reverse_course_url('course_handler', course_key),
            'import_status_url': reverse_course_url("import_status_handler", course_key, kwargs={'filename': "fillerName"}),
        })
    else:
        return HttpResponseNotFound()


# pylint: disable=unused-argument
@require_GET
@ensure_csrf_cookie
@login_required
def import_status_handler(request, course_key_string, filename=None):
    """
    Returns an integer corresponding to the status of a file import. These are:

        0 : No status info found (import done or upload still in progress)
        1 : Extracting file
        2 : Validating.
        3 : Importing to mongo

    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    try:
        session_status = request.session["import_status"]
        status = session_status[course_key_string + filename]
    except KeyError:
        status = 0

    return JsonResponse({"ImportStatus": status})


# pylint: disable=unused-argument
@ensure_csrf_cookie
@login_required
@require_http_methods(("GET",))
def export_handler(request, course_key_string):
    """
    The restful handler for exporting a course.

    GET
        html: return html page for import page
        application/x-tgz: return tar.gz file containing exported course
        json: not supported

    Note that there are 2 ways to request the tar.gz file. The request header can specify
    application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).

    If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
    which describes the error.
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)

    # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
    requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))

    export_url = reverse_course_url('export_handler', course_key) + '?_accept=application/x-tgz'
    if 'application/x-tgz' in requested_format:
        name = course_module.url_name
        export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
        root_dir = path(mkdtemp())

        try:
            export_to_xml(modulestore('direct'), contentstore(), course_module.id, root_dir, name, modulestore())

            logging.debug('tar file being generated at {0}'.format(export_file.name))
            with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
                tar_file.add(root_dir / name, arcname=name)
        except SerializationError as exc:
            log.exception('There was an error exporting course %s', course_module.id)
            unit = None
            failed_item = None
            parent = None
            try:
                failed_item = modulestore().get_item(exc.location)
                parent_locs = modulestore().get_parent_locations(failed_item.location)

                if len(parent_locs) > 0:
                    parent = modulestore().get_item(parent_locs[0])
                    if parent.location.category == 'vertical':
                        unit = parent
            except:  # pylint: disable=bare-except
                # if we have a nested exception, then we'll show the more generic error message
                pass

            return render_to_response('export.html', {
                'context_course': course_module,
                'in_err': True,
                'raw_err_msg': str(exc),
                'failed_module': failed_item,
                'unit': unit,
                'edit_unit_url': reverse_usage_url("unit_handler", parent.location) if parent else "",
                'course_home_url': reverse_course_url("course_handler", course_key),
                'export_url': export_url
            })
        except Exception as exc:
            log.exception('There was an error exporting course %s', course_module.id)
            return render_to_response('export.html', {
                'context_course': course_module,
                'in_err': True,
                'unit': None,
                'raw_err_msg': str(exc),
                'course_home_url': reverse_course_url("course_handler", course_key),
                'export_url': export_url
            })
        finally:
            shutil.rmtree(root_dir / name)

        wrapper = FileWrapper(export_file)
        response = HttpResponse(wrapper, content_type='application/x-tgz')
        response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
        response['Content-Length'] = os.path.getsize(export_file.name)
        return response

    elif 'text/html' in requested_format:
        return render_to_response('export.html', {
            'context_course': course_module,
            'export_url': export_url
        })

    else:
        # Only HTML or x-tgz request formats are supported (no JSON).
        return HttpResponse(status=406)

########NEW FILE########
__FILENAME__ = item
"""Views for items (modules)."""
from __future__ import absolute_import

import hashlib
import logging
from uuid import uuid4

from collections import OrderedDict
from functools import partial
from static_replace import replace_static_urls
from xmodule_modifiers import wrap_xblock

from django.core.exceptions import PermissionDenied
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseBadRequest, HttpResponse, Http404
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_http_methods

from xblock.fields import Scope
from xblock.fragment import Fragment

import xmodule
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError, DuplicateItemError
from xmodule.modulestore.inheritance import own_metadata
from xmodule.video_module import manage_video_subtitles_save

from util.json_request import expect_json, JsonResponse
from util.string_utils import str_to_bool

from ..utils import get_modulestore

from .access import has_course_access
from .helpers import _xmodule_recurse, xblock_has_own_studio_page
from contentstore.utils import compute_publish_state, PublishState
from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES
from contentstore.views.preview import get_preview_fragment
from edxmako.shortcuts import render_to_string
from models.settings.course_grading import CourseGradingModel
from cms.lib.xblock.runtime import handler_url, local_resource_url
from xmodule.modulestore.keys import UsageKey, CourseKey

__all__ = ['orphan_handler', 'xblock_handler', 'xblock_view_handler']

log = logging.getLogger(__name__)

CREATE_IF_NOT_FOUND = ['course_info']


# In order to allow descriptors to use a handler url, we need to
# monkey-patch the x_module library.
# TODO: Remove this code when Runtimes are no longer created by modulestores
xmodule.x_module.descriptor_global_handler_url = handler_url
xmodule.x_module.descriptor_global_local_resource_url = local_resource_url


def hash_resource(resource):
    """
    Hash a :class:`xblock.fragment.FragmentResource`.
    """
    md5 = hashlib.md5()
    md5.update(repr(resource))
    return md5.hexdigest()


# pylint: disable=unused-argument
@require_http_methods(("DELETE", "GET", "PUT", "POST"))
@login_required
@expect_json
def xblock_handler(request, usage_key_string):
    """
    The restful handler for xblock requests.

    DELETE
        json: delete this xblock instance from the course. Supports query parameters "recurse" to delete
        all children and "all_versions" to delete from all (mongo) versions.
    GET
        json: returns representation of the xblock (locator id, data, and metadata).
              if ?fields=graderType, it returns the graderType for the unit instead of the above.
        html: returns HTML for rendering the xblock (which includes both the "preview" view and the "editor" view)
    PUT or POST
        json: if xblock locator is specified, update the xblock instance. The json payload can contain
              these fields, all optional:
                :data: the new value for the data.
                :children: the unicode representation of the UsageKeys of children for this xblock.
                :metadata: new values for the metadata fields. Any whose values are None will be deleted not set
                       to None! Absent ones will be left alone.
                :nullout: which metadata fields to set to None
                :graderType: change how this unit is graded
                :publish: can be one of three values, 'make_public, 'make_private', or 'create_draft'
              The JSON representation on the updated xblock (minus children) is returned.

              if usage_key_string is not specified, create a new xblock instance, either by duplicating
              an existing xblock, or creating an entirely new one. The json playload can contain
              these fields:
                :parent_locator: parent for new xblock, required for both duplicate and create new instance
                :duplicate_source_locator: if present, use this as the source for creating a duplicate copy
                :category: type of xblock, required if duplicate_source_locator is not present.
                :display_name: name for new xblock, optional
                :boilerplate: template name for populating fields, optional and only used
                     if duplicate_source_locator is not present
              The locator (unicode representation of a UsageKey) for the created xblock (minus children) is returned.
    """
    if usage_key_string:
        usage_key = UsageKey.from_string(usage_key_string)
        if not has_course_access(request.user, usage_key.course_key):
            raise PermissionDenied()

        if request.method == 'GET':
            accept_header = request.META.get('HTTP_ACCEPT', 'application/json')

            if 'application/json' in accept_header:
                fields = request.REQUEST.get('fields', '').split(',')
                if 'graderType' in fields:
                    # right now can't combine output of this w/ output of _get_module_info, but worthy goal
                    return JsonResponse(CourseGradingModel.get_section_grader_type(usage_key))
                # TODO: pass fields to _get_module_info and only return those
                rsp = _get_module_info(usage_key)
                return JsonResponse(rsp)
            else:
                return HttpResponse(status=406)

        elif request.method == 'DELETE':
            delete_children = str_to_bool(request.REQUEST.get('recurse', 'False'))
            delete_all_versions = str_to_bool(request.REQUEST.get('all_versions', 'False'))

            return _delete_item_at_location(usage_key, delete_children, delete_all_versions, request.user)
        else:  # Since we have a usage_key, we are updating an existing xblock.
            return _save_item(
                request,
                usage_key,
                data=request.json.get('data'),
                children=request.json.get('children'),
                metadata=request.json.get('metadata'),
                nullout=request.json.get('nullout'),
                grader_type=request.json.get('graderType'),
                publish=request.json.get('publish'),
            )
    elif request.method in ('PUT', 'POST'):
        if 'duplicate_source_locator' in request.json:
            parent_usage_key = UsageKey.from_string(request.json['parent_locator'])
            duplicate_source_usage_key = UsageKey.from_string(request.json['duplicate_source_locator'])

            dest_usage_key = _duplicate_item(
                parent_usage_key,
                duplicate_source_usage_key,
                request.json.get('display_name'),
                request.user,
            )

            return JsonResponse({"locator": unicode(dest_usage_key)})
        else:
            return _create_item(request)
    else:
        return HttpResponseBadRequest(
            "Only instance creation is supported without a usage key.",
            content_type="text/plain"
        )

# pylint: disable=unused-argument
@require_http_methods(("GET"))
@login_required
@expect_json
def xblock_view_handler(request, usage_key_string, view_name):
    """
    The restful handler for requests for rendered xblock views.

    Returns a json object containing two keys:
        html: The rendered html of the view
        resources: A list of tuples where the first element is the resource hash, and
            the second is the resource description
    """
    usage_key = UsageKey.from_string(usage_key_string)
    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    accept_header = request.META.get('HTTP_ACCEPT', 'application/json')

    if 'application/json' in accept_header:
        store = get_modulestore(usage_key)
        xblock = store.get_item(usage_key)
        is_read_only = _is_xblock_read_only(xblock)
        container_views = ['container_preview', 'reorderable_container_child_preview']
        unit_views = ['student_view']

        # wrap the generated fragment in the xmodule_editor div so that the javascript
        # can bind to it correctly
        xblock.runtime.wrappers.append(partial(wrap_xblock, 'StudioRuntime', usage_id_serializer=unicode))

        if view_name == 'studio_view':
            try:
                fragment = xblock.render('studio_view')
            # catch exceptions indiscriminately, since after this point they escape the
            # dungeon and surface as uneditable, unsaveable, and undeletable
            # component-goblins.
            except Exception as exc:                          # pylint: disable=w0703
                log.debug("unable to render studio_view for %r", xblock, exc_info=True)
                fragment = Fragment(render_to_string('html_error.html', {'message': str(exc)}))

            # change not authored by requestor but by xblocks.
            store.update_item(xblock, None)

        elif view_name == 'student_view' and xblock_has_own_studio_page(xblock):
            context = {
                'runtime_type': 'studio',
                'container_view': False,
                'read_only': is_read_only,
                'root_xblock': xblock,
            }
            # For non-leaf xblocks on the unit page, show the special rendering
            # which links to the new container page.
            html = render_to_string('container_xblock_component.html', {
                'xblock_context': context,
                'xblock': xblock,
                'locator': usage_key,
            })
            return JsonResponse({
                'html': html,
                'resources': [],
            })
        elif view_name in (unit_views + container_views):
            is_container_view = (view_name in container_views)

            # Determine the items to be shown as reorderable. Note that the view
            # 'reorderable_container_child_preview' is only rendered for xblocks that
            # are being shown in a reorderable container, so the xblock is automatically
            # added to the list.
            reorderable_items = set()
            if view_name == 'reorderable_container_child_preview':
                reorderable_items.add(xblock.location)

            # Only show the new style HTML for the container view, i.e. for non-verticals
            # Note: this special case logic can be removed once the unit page is replaced
            # with the new container view.
            context = {
                'runtime_type': 'studio',
                'container_view': is_container_view,
                'read_only': is_read_only,
                'root_xblock': xblock if (view_name == 'container_preview') else None,
                'reorderable_items': reorderable_items
            }

            fragment = get_preview_fragment(request, xblock, context)
            # For old-style pages (such as unit and static pages), wrap the preview with
            # the component div. Note that the container view recursively adds headers
            # into the preview fragment, so we don't want to add another header here.
            if not is_container_view:
                fragment.content = render_to_string('component.html', {
                    'xblock_context': context,
                    'preview': fragment.content,
                    'label': xblock.display_name or xblock.scope_ids.block_type,
                })
        else:
            raise Http404

        hashed_resources = OrderedDict()
        for resource in fragment.resources:
            hashed_resources[hash_resource(resource)] = resource

        return JsonResponse({
            'html': fragment.content,
            'resources': hashed_resources.items()
        })

    else:
        return HttpResponse(status=406)


def _is_xblock_read_only(xblock):
    """
    Returns true if the specified xblock is read-only, meaning that it cannot be edited.
    """
    # We allow direct editing of xblocks in DIRECT_ONLY_CATEGORIES (for example, static pages).
    if xblock.category in DIRECT_ONLY_CATEGORIES:
        return False
    component_publish_state = compute_publish_state(xblock)
    return component_publish_state == PublishState.public


def _save_item(request, usage_key, data=None, children=None, metadata=None, nullout=None,
               grader_type=None, publish=None):
    """
    Saves xblock w/ its fields. Has special processing for grader_type, publish, and nullout and Nones in metadata.
    nullout means to truly set the field to None whereas nones in metadata mean to unset them (so they revert
    to default).
    """
    store = get_modulestore(usage_key)

    try:
        existing_item = store.get_item(usage_key)
    except ItemNotFoundError:
        if usage_key.category in CREATE_IF_NOT_FOUND:
            # New module at this location, for pages that are not pre-created.
            # Used for course info handouts.
            store.create_and_save_xmodule(usage_key)
            existing_item = store.get_item(usage_key)
        else:
            raise
    except InvalidLocationError:
        log.error("Can't find item by location.")
        return JsonResponse({"error": "Can't find item by location: " + unicode(usage_key)}, 404)

    old_metadata = own_metadata(existing_item)

    if publish:
        if publish == 'make_private':
            _xmodule_recurse(
                existing_item,
                lambda i: modulestore().unpublish(i.location),
                ignore_exception=ItemNotFoundError
            )
        elif publish == 'create_draft':
            # This recursively clones the existing item location to a draft location (the draft is
            # implicit, because modulestore is a Draft modulestore)
            _xmodule_recurse(
                existing_item,
                lambda i: modulestore().convert_to_draft(i.location),
                ignore_exception=DuplicateItemError
            )

    if data:
        # TODO Allow any scope.content fields not just "data" (exactly like the get below this)
        existing_item.data = data
    else:
        data = existing_item.get_explicitly_set_fields_by_scope(Scope.content)

    if children is not None:
        children_usage_keys = [
            UsageKey.from_string(child)
            for child
            in children
        ]
        existing_item.children = children_usage_keys

    # also commit any metadata which might have been passed along
    if nullout is not None or metadata is not None:
        # the postback is not the complete metadata, as there's system metadata which is
        # not presented to the end-user for editing. So let's use the original (existing_item) and
        # 'apply' the submitted metadata, so we don't end up deleting system metadata.
        if nullout is not None:
            for metadata_key in nullout:
                setattr(existing_item, metadata_key, None)

        # update existing metadata with submitted metadata (which can be partial)
        # IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
        # the intent is to make it None, use the nullout field
        if metadata is not None:
            for metadata_key, value in metadata.items():
                field = existing_item.fields[metadata_key]

                if value is None:
                    field.delete_from(existing_item)
                else:
                    try:
                        value = field.from_json(value)
                    except ValueError:
                        return JsonResponse({"error": "Invalid data"}, 400)
                    field.write_to(existing_item, value)

        if existing_item.category == 'video':
            manage_video_subtitles_save(existing_item, request.user, old_metadata, generate_translation=True)

    # commit to datastore
    store.update_item(existing_item, request.user.id)

    result = {
        'id': unicode(usage_key),
        'data': data,
        'metadata': own_metadata(existing_item)
    }

    if grader_type is not None:
        result.update(CourseGradingModel.update_section_grader_type(existing_item, grader_type, request.user))

    # Make public after updating the xblock, in case the caller asked
    # for both an update and a publish.
    if publish and publish == 'make_public':
        def _publish(block):
            # This is super gross, but prevents us from publishing something that
            # we shouldn't. Ideally, all modulestores would have a consistant
            # interface for publishing. However, as of now, only the DraftMongoModulestore
            # does, so we have to check for the attribute explicitly.
            store = get_modulestore(block.location)
            store.publish(block.location, request.user.id)

        _xmodule_recurse(
            existing_item,
            _publish
        )

    # Note that children aren't being returned until we have a use case.
    return JsonResponse(result)


@login_required
@expect_json
def _create_item(request):
    """View for create items."""
    usage_key = UsageKey.from_string(request.json['parent_locator'])
    category = request.json['category']

    display_name = request.json.get('display_name')

    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    parent = get_modulestore(category).get_item(usage_key)
    dest_usage_key = usage_key.replace(category=category, name=uuid4().hex)

    # get the metadata, display_name, and definition from the request
    metadata = {}
    data = None
    template_id = request.json.get('boilerplate')
    if template_id:
        clz = parent.runtime.load_block_type(category)
        if clz is not None:
            template = clz.get_template(template_id)
            if template is not None:
                metadata = template.get('metadata', {})
                data = template.get('data')

    if display_name is not None:
        metadata['display_name'] = display_name

    get_modulestore(category).create_and_save_xmodule(
        dest_usage_key,
        definition_data=data,
        metadata=metadata,
        system=parent.runtime,
    )

    # TODO replace w/ nicer accessor
    if not 'detached' in parent.runtime.load_block_type(category)._class_tags:
        parent.children.append(dest_usage_key)
        get_modulestore(parent.location).update_item(parent, request.user.id)

    return JsonResponse({"locator": unicode(dest_usage_key), "courseKey": unicode(dest_usage_key.course_key)})


def _duplicate_item(parent_usage_key, duplicate_source_usage_key, display_name=None, user=None):
    """
    Duplicate an existing xblock as a child of the supplied parent_usage_key.
    """
    store = get_modulestore(duplicate_source_usage_key)
    source_item = store.get_item(duplicate_source_usage_key)
    # Change the blockID to be unique.
    dest_usage_key = duplicate_source_usage_key.replace(name=uuid4().hex)
    category = dest_usage_key.category

    # Update the display name to indicate this is a duplicate (unless display name provided).
    duplicate_metadata = own_metadata(source_item)
    if display_name is not None:
        duplicate_metadata['display_name'] = display_name
    else:
        if source_item.display_name is None:
            duplicate_metadata['display_name'] = _("Duplicate of {0}").format(source_item.category)
        else:
            duplicate_metadata['display_name'] = _("Duplicate of '{0}'").format(source_item.display_name)

    get_modulestore(category).create_and_save_xmodule(
        dest_usage_key,
        definition_data=source_item.data if hasattr(source_item, 'data') else None,
        metadata=duplicate_metadata,
        system=source_item.runtime,
    )

    dest_module = get_modulestore(category).get_item(dest_usage_key)
    # Children are not automatically copied over (and not all xblocks have a 'children' attribute).
    # Because DAGs are not fully supported, we need to actually duplicate each child as well.
    if source_item.has_children:
        dest_module.children = []
        for child in source_item.children:
            dupe = _duplicate_item(dest_usage_key, child, user=user)
            dest_module.children.append(dupe)
        get_modulestore(dest_usage_key).update_item(dest_module, user.id if user else None)

    if not 'detached' in source_item.runtime.load_block_type(category)._class_tags:
        parent = get_modulestore(parent_usage_key).get_item(parent_usage_key)
        # If source was already a child of the parent, add duplicate immediately afterward.
        # Otherwise, add child to end.
        if duplicate_source_usage_key in parent.children:
            source_index = parent.children.index(duplicate_source_usage_key)
            parent.children.insert(source_index + 1, dest_usage_key)
        else:
            parent.children.append(dest_usage_key)
        get_modulestore(parent_usage_key).update_item(parent, user.id if user else None)

    return dest_usage_key


def _delete_item_at_location(item_usage_key, delete_children=False, delete_all_versions=False, user=None):
    """
    Deletes the item at with the given Location.

    It is assumed that course permissions have already been checked.
    """
    store = get_modulestore(item_usage_key)

    item = store.get_item(item_usage_key)

    if delete_children:
        _xmodule_recurse(item, lambda i: store.delete_item(i.location, delete_all_versions=delete_all_versions))
    else:
        store.delete_item(item.location, delete_all_versions=delete_all_versions)

    # cdodge: we need to remove our parent's pointer to us so that it is no longer dangling
    if delete_all_versions:
        parent_locs = modulestore('direct').get_parent_locations(item_usage_key)

        for parent_loc in parent_locs:
            parent = modulestore('direct').get_item(parent_loc)
            parent.children.remove(item_usage_key)
            modulestore('direct').update_item(parent, user.id if user else None)

    return JsonResponse()


# pylint: disable=W0613
@login_required
@require_http_methods(("GET", "DELETE"))
def orphan_handler(request, course_key_string):
    """
    View for handling orphan related requests. GET gets all of the current orphans.
    DELETE removes all orphans (requires is_staff access)

    An orphan is a block whose category is not in the DETACHED_CATEGORY list, is not the root, and is not reachable
    from the root via children
    """
    course_usage_key = CourseKey.from_string(course_key_string)
    if request.method == 'GET':
        if has_course_access(request.user, course_usage_key):
            return JsonResponse(modulestore().get_orphans(course_usage_key))
        else:
            raise PermissionDenied()
    if request.method == 'DELETE':
        if request.user.is_staff:
            items = modulestore().get_orphans(course_usage_key)
            for itemloc in items:
                # get_orphans returns the deprecated string format
                usage_key = course_usage_key.make_usage_key_from_deprecated_string(itemloc)
                modulestore('draft').delete_item(usage_key, delete_all_versions=True)
            return JsonResponse({'deleted': items})
        else:
            raise PermissionDenied()


def _get_module_info(usage_key, rewrite_static_links=True):
    """
    metadata, data, id representation of a leaf module fetcher.
    :param usage_key: A UsageKey
    """
    store = get_modulestore(usage_key)
    try:
        module = store.get_item(usage_key)
    except ItemNotFoundError:
        if usage_key.category in CREATE_IF_NOT_FOUND:
            # Create a new one for certain categories only. Used for course info handouts.
            store.create_and_save_xmodule(usage_key)
            module = store.get_item(usage_key)
        else:
            raise

    data = getattr(module, 'data', '')
    if rewrite_static_links:
        data = replace_static_urls(
            data,
            None,
            course_id=usage_key.course_key
        )

    # Note that children aren't being returned until we have a use case.
    return {
        'id': unicode(usage_key),
        'data': data,
        'metadata': own_metadata(module)
    }

########NEW FILE########
__FILENAME__ = preview
from __future__ import absolute_import

import logging
from functools import partial

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import Http404, HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from edxmako.shortcuts import render_to_string

from xmodule_modifiers import replace_static_urls, wrap_xblock, wrap_fragment
from xmodule.error_module import ErrorDescriptor
from xmodule.exceptions import NotFoundError, ProcessingError
from xmodule.modulestore.django import modulestore, ModuleI18nService
from xmodule.modulestore.keys import UsageKey
from xmodule.x_module import ModuleSystem
from xblock.runtime import KvsFieldData
from xblock.django.request import webob_to_django_response, django_to_webob_request
from xblock.exceptions import NoSuchHandlerError
from xblock.fragment import Fragment

from lms.lib.xblock.field_data import LmsFieldData
from cms.lib.xblock.runtime import local_resource_url

from util.sandboxing import can_execute_unsafe_code

import static_replace
from .session_kv_store import SessionKeyValueStore
from .helpers import render_from_lms, xblock_has_own_studio_page

from contentstore.views.access import get_user_role

__all__ = ['preview_handler']

log = logging.getLogger(__name__)


@login_required
def preview_handler(request, usage_key_string, handler, suffix=''):
    """
    Dispatch an AJAX action to an xblock

    usage_key_string: The usage_key_string-id of the block to dispatch to, passed through `quote_slashes`
    handler: The handler to execute
    suffix: The remainder of the url to be passed to the handler
    """
    usage_key = UsageKey.from_string(usage_key_string)

    descriptor = modulestore().get_item(usage_key)
    instance = _load_preview_module(request, descriptor)
    # Let the module handle the AJAX
    req = django_to_webob_request(request)
    try:
        resp = instance.handle(handler, req, suffix)

    except NoSuchHandlerError:
        log.exception("XBlock %s attempted to access missing handler %r", instance, handler)
        raise Http404

    except NotFoundError:
        log.exception("Module indicating to user that request doesn't exist")
        raise Http404

    except ProcessingError:
        log.warning("Module raised an error while processing AJAX request",
                    exc_info=True)
        return HttpResponseBadRequest()

    except Exception:
        log.exception("error processing ajax call")
        raise

    return webob_to_django_response(resp)


class PreviewModuleSystem(ModuleSystem):  # pylint: disable=abstract-method
    """
    An XModule ModuleSystem for use in Studio previews
    """
    # xmodules can check for this attribute during rendering to determine if
    # they are being rendered for preview (i.e. in Studio)
    is_author_mode = True

    def handler_url(self, block, handler_name, suffix='', query='', thirdparty=False):
        return reverse('preview_handler', kwargs={
            'usage_key_string': unicode(block.location),
            'handler': handler_name,
            'suffix': suffix,
        }) + '?' + query

    def local_resource_url(self, block, uri):
        return local_resource_url(block, uri)


def _preview_module_system(request, descriptor):
    """
    Returns a ModuleSystem for the specified descriptor that is specialized for
    rendering module previews.

    request: The active django request
    descriptor: An XModuleDescriptor
    """

    course_id = descriptor.location.course_key
    display_name_only = (descriptor.category == 'static_tab')

    wrappers = [
        # This wrapper wraps the module in the template specified above
        partial(wrap_xblock, 'PreviewRuntime', display_name_only=display_name_only, usage_id_serializer=unicode),

        # This wrapper replaces urls in the output that start with /static
        # with the correct course-specific url for the static content
        partial(replace_static_urls, None, course_id=course_id),
        _studio_wrap_xblock,
    ]

    return PreviewModuleSystem(
        static_url=settings.STATIC_URL,
        # TODO (cpennington): Do we want to track how instructors are using the preview problems?
        track_function=lambda event_type, event: None,
        filestore=descriptor.runtime.resources_fs,
        get_module=partial(_load_preview_module, request),
        render_template=render_from_lms,
        debug=True,
        replace_urls=partial(static_replace.replace_static_urls, data_directory=None, course_id=course_id),
        user=request.user,
        can_execute_unsafe_code=(lambda: can_execute_unsafe_code(course_id)),
        mixins=settings.XBLOCK_MIXINS,
        course_id=course_id,
        anonymous_student_id='student',

        # Set up functions to modify the fragment produced by student_view
        wrappers=wrappers,
        error_descriptor_class=ErrorDescriptor,
        get_user_role=lambda: get_user_role(request.user, course_id),
        descriptor_runtime=descriptor.runtime,
        services={
            "i18n": ModuleI18nService(),
        },
    )


def _load_preview_module(request, descriptor):
    """
    Return a preview XModule instantiated from the supplied descriptor.

    request: The active django request
    descriptor: An XModuleDescriptor
    """
    student_data = KvsFieldData(SessionKeyValueStore(request))
    descriptor.bind_for_student(
        _preview_module_system(request, descriptor),
        LmsFieldData(descriptor._field_data, student_data),  # pylint: disable=protected-access
    )
    return descriptor


def _is_xblock_reorderable(xblock, context):
    """
    Returns true if the specified xblock is in the set of reorderable xblocks.
    """
    return xblock.location in context['reorderable_items']


# pylint: disable=unused-argument
def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False):
    """
    Wraps the results of rendering an XBlock view in a div which adds a header and Studio action buttons.
    """
    # Only add the Studio wrapper when on the container page. The unit page will remain as is for now.
    if context.get('container_view', None) and view == 'student_view':
        root_xblock = context.get('root_xblock')
        is_root = root_xblock and xblock.location == root_xblock.location
        is_reorderable = _is_xblock_reorderable(xblock, context)
        template_context = {
            'xblock_context': context,
            'xblock': xblock,
            'content': frag.content,
            'is_root': is_root,
            'is_reorderable': is_reorderable,
        }
        # For child xblocks with their own page, render a link to the page
        if xblock_has_own_studio_page(xblock) and not is_root:
            template = 'studio_container_wrapper.html'
        else:
            template = 'studio_xblock_wrapper.html'
        html = render_to_string(template, template_context)
        frag = wrap_fragment(frag, html)
    return frag


def get_preview_fragment(request, descriptor, context):
    """
    Returns the HTML returned by the XModule's student_view,
    specified by the descriptor and idx.
    """
    module = _load_preview_module(request, descriptor)

    try:
        fragment = module.render("student_view", context)
    except Exception as exc:                          # pylint: disable=W0703
        log.warning("Unable to render student_view for %r", module, exc_info=True)
        fragment = Fragment(render_to_string('html_error.html', {'message': str(exc)}))
    return fragment

########NEW FILE########
__FILENAME__ = public
"""
Public views
"""
from django_future.csrf import ensure_csrf_cookie
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.shortcuts import redirect
from django.conf import settings

from edxmako.shortcuts import render_to_response

from external_auth.views import (ssl_login_shortcut, ssl_get_cert_from_request,
                                 redirect_with_get)
from microsite_configuration import microsite

__all__ = ['signup', 'login_page', 'howitworks']


@ensure_csrf_cookie
def signup(request):
    """
    Display the signup form.
    """
    csrf_token = csrf(request)['csrf_token']
    if request.user.is_authenticated():
        return redirect('/course/')
    if settings.FEATURES.get('AUTH_USE_CERTIFICATES_IMMEDIATE_SIGNUP'):
        # Redirect to course to login to process their certificate if SSL is enabled
        # and registration is disabled.
        return redirect_with_get('login', request.GET, False)

    return render_to_response('register.html', {'csrf': csrf_token})


@ssl_login_shortcut
@ensure_csrf_cookie
def login_page(request):
    """
    Display the login form.
    """
    csrf_token = csrf(request)['csrf_token']
    if (settings.FEATURES['AUTH_USE_CERTIFICATES'] and
            ssl_get_cert_from_request(request)):
        # SSL login doesn't require a login view, so redirect
        # to course now that the user is authenticated via
        # the decorator.
        next_url = request.GET.get('next')
        if next_url:
            return redirect(next_url)
        else:
            return redirect('/course/')
    if settings.FEATURES.get('AUTH_USE_CAS'):
        # If CAS is enabled, redirect auth handling to there
        return redirect(reverse('cas-login'))

    return render_to_response(
        'login.html',
        {
            'csrf': csrf_token,
            'forgot_password_link': "//{base}/login#forgot-password-modal".format(base=settings.LMS_BASE),
            'platform_name': microsite.get_value('platform_name', settings.PLATFORM_NAME),
        }
    )


def howitworks(request):
    "Proxy view"
    if request.user.is_authenticated():
        return redirect('/course/')
    else:
        return render_to_response('howitworks.html', {})

########NEW FILE########
__FILENAME__ = session_kv_store
"""
An :class:`~xblock.runtime.KeyValueStore` that stores data in the django session
"""
from __future__ import absolute_import

from xblock.runtime import KeyValueStore

class SessionKeyValueStore(KeyValueStore):
    def __init__(self, request):
        self._session = request.session

    def get(self, key):
        return self._session[tuple(key)]

    def set(self, key, value):
        self._session[tuple(key)] = value

    def delete(self, key):
        del self._session[tuple(key)]

    def has(self, key):
        return tuple(key) in self._session

########NEW FILE########
__FILENAME__ = tabs
"""
Views related to course tabs
"""
from access import has_course_access
from util.json_request import expect_json, JsonResponse

from django.http import HttpResponseNotFound
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django_future.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_http_methods
from edxmako.shortcuts import render_to_response
from xmodule.modulestore.django import modulestore
from xmodule.tabs import CourseTabList, StaticTab, CourseTab, InvalidTabsException
from xmodule.modulestore.keys import CourseKey, UsageKey

from ..utils import get_lms_link_for_item

__all__ = ['tabs_handler']

@expect_json
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
def tabs_handler(request, course_key_string):
    """
    The restful handler for static tabs.

    GET
        html: return page for editing static tabs
        json: not supported
    PUT or POST
        json: update the tab order. It is expected that the request body contains a JSON-encoded dict with entry "tabs".
        The value for "tabs" is an array of tab locators, indicating the desired order of the tabs.

    Creating a tab, deleting a tab, or changing its contents is not supported through this method.
    Instead use the general xblock URL (see item.xblock_handler).
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_item = modulestore().get_course(course_key)

    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            raise NotImplementedError('coming soon')
        else:
            if 'tabs' in request.json:
                return reorder_tabs_handler(course_item, request)
            elif 'tab_id_locator' in request.json:
                return edit_tab_handler(course_item, request)
            else:
                raise NotImplementedError('Creating or changing tab content is not supported.')

    elif request.method == 'GET':  # assume html
        # get all tabs from the tabs list: static tabs (a.k.a. user-created tabs) and built-in tabs
        # present in the same order they are displayed in LMS

        tabs_to_render = []
        for tab in CourseTabList.iterate_displayable_cms(
                course_item,
                settings,
        ):
            if isinstance(tab, StaticTab):
                # static tab needs its locator information to render itself as an xmodule
                static_tab_loc = course_key.make_usage_key('static_tab', tab.url_slug)
                tab.locator = static_tab_loc
            tabs_to_render.append(tab)

        return render_to_response('edit-tabs.html', {
            'context_course': course_item,
            'tabs_to_render': tabs_to_render,
            'lms_link': get_lms_link_for_item(course_item.location),
        })
    else:
        return HttpResponseNotFound()


def reorder_tabs_handler(course_item, request):
    """
    Helper function for handling reorder of tabs request
    """

    # Tabs are identified by tab_id or locators.
    # The locators are used to identify static tabs since they are xmodules.
    # Although all tabs have tab_ids, newly created static tabs do not know
    # their tab_ids since the xmodule editor uses only locators to identify new objects.
    requested_tab_id_locators = request.json['tabs']

    # original tab list in original order
    old_tab_list = course_item.tabs

    # create a new list in the new order
    new_tab_list = []
    for tab_id_locator in requested_tab_id_locators:
        tab = get_tab_by_tab_id_locator(old_tab_list, tab_id_locator)
        if tab is None:
            return JsonResponse(
                {"error": "Tab with id_locator '{0}' does not exist.".format(tab_id_locator)}, status=400
            )
        new_tab_list.append(tab)

    # the old_tab_list may contain additional tabs that were not rendered in the UI because of
    # global or course settings.  so add those to the end of the list.
    non_displayed_tabs = set(old_tab_list) - set(new_tab_list)
    new_tab_list.extend(non_displayed_tabs)

    # validate the tabs to make sure everything is Ok (e.g., did the client try to reorder unmovable tabs?)
    try:
        CourseTabList.validate_tabs(new_tab_list)
    except InvalidTabsException, exception:
        return JsonResponse(
            {"error": "New list of tabs is not valid: {0}.".format(str(exception))}, status=400
        )

    # persist the new order of the tabs
    course_item.tabs = new_tab_list
    modulestore('direct').update_item(course_item, request.user.id)

    return JsonResponse()


def edit_tab_handler(course_item, request):
    """
    Helper function for handling requests to edit settings of a single tab
    """

    # Tabs are identified by tab_id or locator
    tab_id_locator = request.json['tab_id_locator']

    # Find the given tab in the course
    tab = get_tab_by_tab_id_locator(course_item.tabs, tab_id_locator)
    if tab is None:
        return JsonResponse(
            {"error": "Tab with id_locator '{0}' does not exist.".format(tab_id_locator)}, status=400
        )

    if 'is_hidden' in request.json:
        # set the is_hidden attribute on the requested tab
        tab.is_hidden = request.json['is_hidden']
        modulestore('direct').update_item(course_item, request.user.id)
    else:
        raise NotImplementedError('Unsupported request to edit tab: {0}'.format(request.json))

    return JsonResponse()


def get_tab_by_tab_id_locator(tab_list, tab_id_locator):
    """
    Look for a tab with the specified tab_id or locator.  Returns the first matching tab.
    """
    if 'tab_id' in tab_id_locator:
        tab = CourseTabList.get_tab_by_id(tab_list, tab_id_locator['tab_id'])
    elif 'tab_locator' in tab_id_locator:
        tab = get_tab_by_locator(tab_list, tab_id_locator['tab_locator'])
    return tab


def get_tab_by_locator(tab_list, usage_key_string):
    """
    Look for a tab with the specified locator.  Returns the first matching tab.
    """
    tab_location = UsageKey.from_string(usage_key_string)
    item = modulestore('direct').get_item(tab_location)
    static_tab = StaticTab(
        name=item.display_name,
        url_slug=item.location.name,
    )
    return CourseTabList.get_tab_by_id(tab_list, static_tab.tab_id)


# "primitive" tab edit functions driven by the command line.
# These should be replaced/deleted by a more capable GUI someday.
# Note that the command line UI identifies the tabs with 1-based
# indexing, but this implementation code is standard 0-based.

def validate_args(num, tab_type):
    "Throws for the disallowed cases."
    if num <= 1:
        raise ValueError('Tabs 1 and 2 cannot be edited')
    if tab_type == 'static_tab':
        raise ValueError('Tabs of type static_tab cannot be edited here (use Studio)')


def primitive_delete(course, num):
    "Deletes the given tab number (0 based)."
    tabs = course.tabs
    validate_args(num, tabs[num].get('type', ''))
    del tabs[num]
    # Note for future implementations: if you delete a static_tab, then Chris Dodge
    # points out that there's other stuff to delete beyond this element.
    # This code happens to not delete static_tab so it doesn't come up.
    modulestore('direct').update_item(course, '**replace_user**')


def primitive_insert(course, num, tab_type, name):
    "Inserts a new tab at the given number (0 based)."
    validate_args(num, tab_type)
    new_tab = CourseTab.from_json({u'type': unicode(tab_type), u'name': unicode(name)})
    tabs = course.tabs
    tabs.insert(num, new_tab)
    modulestore('direct').update_item(course, '**replace_user**')


########NEW FILE########
__FILENAME__ = test_access
"""
Tests access.py
"""
from django.test import TestCase
from django.contrib.auth.models import User

from student.roles import CourseInstructorRole, CourseStaffRole
from student.tests.factories import AdminFactory
from student.auth import add_users
from contentstore.views.access import get_user_role
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class RolesTest(TestCase):
    """
    Tests for lti user role serialization.
    """
    def setUp(self):
        """ Test case setup """
        self.global_admin = AdminFactory()
        self.instructor = User.objects.create_user('testinstructor', 'testinstructor+courses@edx.org', 'foo')
        self.staff = User.objects.create_user('teststaff', 'teststaff+courses@edx.org', 'foo')
        self.course_key = SlashSeparatedCourseKey('mitX', '101', 'test')

    def test_get_user_role_instructor(self):
        """
        Verifies if user is instructor.
        """
        add_users(self.global_admin, CourseInstructorRole(self.course_key), self.instructor)
        self.assertEqual(
            'instructor',
            get_user_role(self.instructor, self.course_key)
        )
        add_users(self.global_admin, CourseStaffRole(self.course_key), self.staff)
        self.assertEqual(
            'instructor',
            get_user_role(self.instructor, self.course_key)
        )

    def test_get_user_role_staff(self):
        """
        Verifies if user is staff.
        """
        add_users(self.global_admin, CourseStaffRole(self.course_key), self.staff)
        self.assertEqual(
            'staff',
            get_user_role(self.staff, self.course_key)
        )

########NEW FILE########
__FILENAME__ = test_assets
"""
Unit tests for the asset upload endpoint.
"""

# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0212

from datetime import datetime
from io import BytesIO
from pytz import UTC
import json
from contentstore.tests.utils import CourseTestCase
from contentstore.views import assets
from contentstore.utils import reverse_course_url
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import import_from_xml
from django.test.utils import override_settings
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation


class AssetsTestCase(CourseTestCase):
    """
    Parent class for all asset tests.
    """
    def setUp(self):
        super(AssetsTestCase, self).setUp()
        self.url = reverse_course_url('assets_handler', self.course.id)

    def upload_asset(self, name="asset-1"):
        f = BytesIO(name)
        f.name = name + ".txt"
        return self.client.post(self.url, {"name": name, "file": f})


class BasicAssetsTestCase(AssetsTestCase):
    def test_basic(self):
        resp = self.client.get(self.url, HTTP_ACCEPT='text/html')
        self.assertEquals(resp.status_code, 200)

    def test_static_url_generation(self):

        course_key = SlashSeparatedCourseKey('org', 'class', 'run')
        location = course_key.make_asset_key('asset', 'my_file_name.jpg')
        path = StaticContent.get_static_path_from_location(location)
        self.assertEquals(path, '/static/my_file_name.jpg')

    def test_pdf_asset(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(
            module_store,
            'common/test/data/',
            ['toy'],
            static_content_store=contentstore(),
            verbose=True
        )
        course = course_items[0]
        url = reverse_course_url('assets_handler', course.id)

        # Test valid contentType for pdf asset (textbook.pdf)
        resp = self.client.get(url, HTTP_ACCEPT='application/json')
        self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf")
        asset_location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/textbook.pdf')
        content = contentstore().find(asset_location)
        # Check after import textbook.pdf has valid contentType ('application/pdf')

        # Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf'
        self.assertEqual(content.content_type, 'application/pdf')


class PaginationTestCase(AssetsTestCase):
    """
    Tests the pagination of assets returned from the REST API.
    """
    def test_json_responses(self):
        self.upload_asset("asset-1")
        self.upload_asset("asset-2")
        self.upload_asset("asset-3")

        # Verify valid page requests
        self.assert_correct_asset_response(self.url, 0, 3, 3)
        self.assert_correct_asset_response(self.url + "?page_size=2", 0, 2, 3)
        self.assert_correct_asset_response(self.url + "?page_size=2&page=1", 2, 1, 3)
        self.assert_correct_sort_response(self.url, 'date_added', 'asc')
        self.assert_correct_sort_response(self.url, 'date_added', 'desc')
        self.assert_correct_sort_response(self.url, 'display_name', 'asc')
        self.assert_correct_sort_response(self.url, 'display_name', 'desc')

        # Verify querying outside the range of valid pages
        self.assert_correct_asset_response(self.url + "?page_size=2&page=-1", 0, 2, 3)
        self.assert_correct_asset_response(self.url + "?page_size=2&page=2", 2, 1, 3)
        self.assert_correct_asset_response(self.url + "?page_size=3&page=1", 0, 3, 3)

    def assert_correct_asset_response(self, url, expected_start, expected_length, expected_total):
        resp = self.client.get(url, HTTP_ACCEPT='application/json')
        json_response = json.loads(resp.content)
        assets = json_response['assets']
        self.assertEquals(json_response['start'], expected_start)
        self.assertEquals(len(assets), expected_length)
        self.assertEquals(json_response['totalCount'], expected_total)

    def assert_correct_sort_response(self, url, sort, direction):
        resp = self.client.get(url + '?sort=' + sort + '&direction=' + direction, HTTP_ACCEPT='application/json')
        json_response = json.loads(resp.content)
        assets = json_response['assets']
        name1 = assets[0][sort]
        name2 = assets[1][sort]
        name3 = assets[2][sort]
        if direction == 'asc':
            self.assertLessEqual(name1, name2)
            self.assertLessEqual(name2, name3)
        else:
            self.assertGreaterEqual(name1, name2)
            self.assertGreaterEqual(name2, name3)


class UploadTestCase(AssetsTestCase):
    """
    Unit tests for uploading a file
    """
    def setUp(self):
        super(UploadTestCase, self).setUp()
        self.url = reverse_course_url('assets_handler', self.course.id)

    def test_happy_path(self):
        resp = self.upload_asset()
        self.assertEquals(resp.status_code, 200)

    def test_no_file(self):
        resp = self.client.post(self.url, {"name": "file.txt"}, "application/json")
        self.assertEquals(resp.status_code, 400)


class AssetToJsonTestCase(AssetsTestCase):
    """
    Unit test for transforming asset information into something
    we can send out to the client via JSON.
    """
    @override_settings(LMS_BASE="lms_base_url")
    def test_basic(self):
        upload_date = datetime(2013, 6, 1, 10, 30, tzinfo=UTC)

        course_key = SlashSeparatedCourseKey('org', 'class', 'run')
        location = course_key.make_asset_key('asset', 'my_file_name.jpg')
        thumbnail_location = course_key.make_asset_key('thumbnail', 'my_file_name_thumb.jpg')

        output = assets._get_asset_json("my_file", upload_date, location, thumbnail_location, True)

        self.assertEquals(output["display_name"], "my_file")
        self.assertEquals(output["date_added"], "Jun 01, 2013 at 10:30 UTC")
        self.assertEquals(output["url"], "/c4x/org/class/asset/my_file_name.jpg")
        self.assertEquals(output["external_url"], "lms_base_url/c4x/org/class/asset/my_file_name.jpg")
        self.assertEquals(output["portable_url"], "/static/my_file_name.jpg")
        self.assertEquals(output["thumbnail"], "/c4x/org/class/thumbnail/my_file_name_thumb.jpg")
        self.assertEquals(output["id"], unicode(location))
        self.assertEquals(output['locked'], True)

        output = assets._get_asset_json("name", upload_date, location, None, False)
        self.assertIsNone(output["thumbnail"])


class LockAssetTestCase(AssetsTestCase):
    """
    Unit test for locking and unlocking an asset.
    """

    def test_locking(self):
        """
        Tests a simple locking and unlocking of an asset in the toy course.
        """
        def verify_asset_locked_state(locked):
            """ Helper method to verify lock state in the contentstore """
            asset_location = StaticContent.get_location_from_path('/c4x/edX/toy/asset/sample_static.txt')
            content = contentstore().find(asset_location)
            self.assertEqual(content.locked, locked)

        def post_asset_update(lock, course):
            """ Helper method for posting asset update. """
            upload_date = datetime(2013, 6, 1, 10, 30, tzinfo=UTC)
            asset_location = course.id.make_asset_key('asset', 'sample_static.txt')
            url = reverse_course_url('assets_handler', course.id, kwargs={'asset_key_string': unicode(asset_location)})

            resp = self.client.post(
                url,
                json.dumps(assets._get_asset_json("sample_static.txt", upload_date, asset_location, None, lock)),
                "application/json"
            )
            self.assertEqual(resp.status_code, 201)
            return json.loads(resp.content)

        # Load the toy course.
        module_store = modulestore('direct')
        _, course_items = import_from_xml(
            module_store,
            'common/test/data/',
            ['toy'],
            static_content_store=contentstore(),
            verbose=True
        )
        course = course_items[0]
        verify_asset_locked_state(False)

        # Lock the asset
        resp_asset = post_asset_update(True, course)
        self.assertTrue(resp_asset['locked'])
        verify_asset_locked_state(True)

        # Unlock the asset
        resp_asset = post_asset_update(False, course)
        self.assertFalse(resp_asset['locked'])
        verify_asset_locked_state(False)

########NEW FILE########
__FILENAME__ = test_checklists
""" Unit tests for checklist methods in views.py. """
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.views.checklist import expand_checklist_action_url
from xmodule.modulestore.tests.factories import CourseFactory

import json
from contentstore.tests.utils import CourseTestCase


class ChecklistTestCase(CourseTestCase):
    """ Test for checklist get and put methods. """
    def setUp(self):
        """ Creates the test course. """
        super(ChecklistTestCase, self).setUp()
        self.course = CourseFactory.create(org='mitX', number='333', display_name='Checklists Course')
        self.checklists_url = self.get_url()

    def get_url(self, checklist_index=None):
        url_args = {'checklist_index': checklist_index} if checklist_index else None
        return reverse_course_url('checklists_handler', self.course.id, kwargs=url_args)

    def get_persisted_checklists(self):
        """ Returns the checklists as persisted in the modulestore. """
        modulestore = get_modulestore(self.course.location)
        return modulestore.get_item(self.course.location).checklists

    def compare_checklists(self, persisted, request):
        """
        Handles url expansion as possible difference and descends into guts
        """
        self.assertEqual(persisted['short_description'], request['short_description'])
        expanded_checklist = expand_checklist_action_url(self.course, persisted)
        for pers, req in zip(expanded_checklist['items'], request['items']):
            self.assertEqual(pers['short_description'], req['short_description'])
            self.assertEqual(pers['long_description'], req['long_description'])
            self.assertEqual(pers['is_checked'], req['is_checked'])
            self.assertEqual(pers['action_url'], req['action_url'])
            self.assertEqual(pers['action_text'], req['action_text'])
            self.assertEqual(pers['action_external'], req['action_external'])

    def test_get_checklists(self):
        """ Tests the get checklists method and URL expansion. """
        response = self.client.get(self.checklists_url)
        self.assertContains(response, "Getting Started With Studio")
        # Verify expansion of action URL happened.
        self.assertContains(response, 'course_team/slashes:mitX+333+Checklists_Course')
        # Verify persisted checklist does NOT have expanded URL.
        checklist_0 = self.get_persisted_checklists()[0]
        self.assertEqual('ManageUsers', get_action_url(checklist_0, 0))
        payload = response.content

        # Now delete the checklists from the course and verify they get repopulated (for courses
        # created before checklists were introduced).
        self.course.checklists = None
        # Save the changed `checklists` to the underlying KeyValueStore before updating the modulestore
        self.course.save()
        modulestore = get_modulestore(self.course.location)
        modulestore.update_item(self.course, self.user.id)
        self.assertEqual(self.get_persisted_checklists(), None)
        response = self.client.get(self.checklists_url)
        self.assertEqual(payload, response.content)

    def test_get_checklists_html(self):
        """ Tests getting the HTML template for the checklists page). """
        response = self.client.get(self.checklists_url, HTTP_ACCEPT='text/html')
        self.assertContains(response, "Getting Started With Studio")
        # The HTML generated will define the handler URL (for use by the Backbone model).
        self.assertContains(response, self.checklists_url)

    def test_update_checklists_no_index(self):
        """ No checklist index, should return all of them. """
        returned_checklists = json.loads(self.client.get(self.checklists_url).content)
        # Verify that persisted checklists do not have expanded action URLs.
        # compare_checklists will verify that returned_checklists DO have expanded action URLs.
        pers = self.get_persisted_checklists()
        self.assertEqual('CourseOutline', get_first_item(pers[1]).get('action_url'))
        for pay, resp in zip(pers, returned_checklists):
            self.compare_checklists(pay, resp)

    def test_update_checklists_index_ignored_on_get(self):
        """ Checklist index ignored on get. """
        update_url = self.get_url(1)

        returned_checklists = json.loads(self.client.get(update_url).content)
        for pay, resp in zip(self.get_persisted_checklists(), returned_checklists):
            self.compare_checklists(pay, resp)

    def test_update_checklists_post_no_index(self):
        """ No checklist index, will error on post. """
        response = self.client.post(self.checklists_url)
        self.assertContains(response, 'Could not save checklist', status_code=400)

    def test_update_checklists_index_out_of_range(self):
        """ Checklist index out of range, will error on post. """
        update_url = self.get_url(100)

        response = self.client.post(update_url)
        self.assertContains(response, 'Could not save checklist', status_code=400)

    def test_update_checklists_index(self):
        """ Check that an update of a particular checklist works. """
        update_url = self.get_url(1)

        payload = self.course.checklists[1]
        self.assertFalse(get_first_item(payload).get('is_checked'))
        self.assertEqual('CourseOutline', get_first_item(payload).get('action_url'))
        get_first_item(payload)['is_checked'] = True

        returned_checklist = json.loads(self.client.ajax_post(update_url, payload).content)
        self.assertTrue(get_first_item(returned_checklist).get('is_checked'))
        persisted_checklist = self.get_persisted_checklists()[1]
        # Verify that persisted checklist does not have expanded action URLs.
        # compare_checklists will verify that returned_checklist DOES have expanded action URLs.
        self.assertEqual('CourseOutline', get_first_item(persisted_checklist).get('action_url'))
        self.compare_checklists(persisted_checklist, returned_checklist)

    def test_update_checklists_delete_unsupported(self):
        """ Delete operation is not supported. """
        update_url = self.get_url(100)
        response = self.client.delete(update_url)
        self.assertEqual(response.status_code, 405)

    def test_expand_checklist_action_url(self):
        """
        Tests the method to expand checklist action url.
        """

        def test_expansion(checklist, index, stored, expanded):
            """
            Tests that the expected expanded value is returned for the item at the given index.

            Also verifies that the original checklist is not modified.
            """
            self.assertEqual(get_action_url(checklist, index), stored)
            expanded_checklist = expand_checklist_action_url(self.course, checklist)
            self.assertEqual(get_action_url(expanded_checklist, index), expanded)
            # Verify no side effect in the original list.
            self.assertEqual(get_action_url(checklist, index), stored)

        test_expansion(self.course.checklists[0], 0, 'ManageUsers', '/course_team/slashes:mitX+333+Checklists_Course/')
        test_expansion(self.course.checklists[1], 1, 'CourseOutline', '/course/slashes:mitX+333+Checklists_Course')
        test_expansion(self.course.checklists[2], 0, 'http://help.edge.edx.org/', 'http://help.edge.edx.org/')


def get_first_item(checklist):
    """ Returns the first item from the checklist. """
    return checklist['items'][0]


def get_action_url(checklist, index):
    """
    Returns the action_url for the item at the specified index in the given checklist.
    """
    return checklist['items'][index]['action_url']

########NEW FILE########
__FILENAME__ = test_container_page
"""
Unit tests for the container page.
"""

import re
from contentstore.utils import compute_publish_state, PublishState
from contentstore.views.tests.utils import StudioPageTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import ItemFactory


class ContainerPageTestCase(StudioPageTestCase):
    """
    Unit tests for the container page.
    """

    container_view = 'container_preview'
    reorderable_child_view = 'reorderable_container_child_preview'

    def setUp(self):
        super(ContainerPageTestCase, self).setUp()
        self.vertical = ItemFactory.create(parent_location=self.sequential.location,
                                           category='vertical', display_name='Unit')
        self.html = ItemFactory.create(parent_location=self.vertical.location,
                                        category="html", display_name="HTML")
        self.child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                  category='split_test', display_name='Split Test')
        self.child_vertical = ItemFactory.create(parent_location=self.child_container.location,
                                                 category='vertical', display_name='Child Vertical')
        self.video = ItemFactory.create(parent_location=self.child_vertical.location,
                                        category="video", display_name="My Video")

    def test_container_html(self):
        self._test_html_content(
            self.child_container,
            expected_section_tag=(
                '<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
                'data-locator="{0}" data-course-key="{0.course_key}">'.format(self.child_container.location)
            ),
            expected_breadcrumbs=(
                r'<a href="/unit/{}"\s*'
                r'class="navigation-link navigation-parent">Unit</a>\s*'
                r'<a href="#" class="navigation-link navigation-current">Split Test</a>'
            ).format(re.escape(unicode(self.vertical.location)))
        )

    def test_container_on_container_html(self):
        """
        Create the scenario of an xblock with children (non-vertical) on the container page.
        This should create a container page that is a child of another container page.
        """
        published_container = ItemFactory.create(
            parent_location=self.child_container.location,
            category="wrapper", display_name="Wrapper"
        )
        ItemFactory.create(
            parent_location=published_container.location,
            category="html", display_name="Child HTML"
        )

        def test_container_html(xblock):
            self._test_html_content(
                xblock,
                expected_section_tag=(
                    '<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
                    'data-locator="{0}" data-course-key="{0.course_key}">'.format(published_container.location)
                ),
                expected_breadcrumbs=(
                    r'<a href="/unit/{unit}"\s*'
                    r'class="navigation-link navigation-parent">Unit</a>\s*'
                    r'<a href="/container/{split_test}"\s*'
                    r'class="navigation-link navigation-parent">Split Test</a>\s*'
                    r'<a href="#" class="navigation-link navigation-current">Wrapper</a>'
                ).format(
                    unit=re.escape(unicode(self.vertical.location)),
                    split_test=re.escape(unicode(self.child_container.location))
                )
            )

        # Test the published version of the container
        test_container_html(published_container)

        # Now make the unit and its children into a draft and validate the container again
        modulestore('draft').convert_to_draft(self.vertical.location)
        modulestore('draft').convert_to_draft(self.child_vertical.location)
        draft_container = modulestore('draft').convert_to_draft(published_container.location)
        test_container_html(draft_container)

    def _test_html_content(self, xblock, expected_section_tag, expected_breadcrumbs):
        """
        Get the HTML for a container page and verify the section tag is correct
        and the breadcrumbs trail is correct.
        """
        html = self.get_page_html(xblock)
        publish_state = compute_publish_state(xblock)
        self.assertIn(expected_section_tag, html)
        # Verify the navigation link at the top of the page is correct.
        self.assertRegexpMatches(html, expected_breadcrumbs)

        # Verify the link that allows users to change publish status.
        expected_message = None
        if publish_state == PublishState.public:
            expected_message = 'you need to edit unit <a href="/unit/{}">Unit</a> as a draft.'
        else:
            expected_message = 'your changes will be published with unit <a href="/unit/{}">Unit</a>.'
        expected_unit_link = expected_message.format(self.vertical.location)
        self.assertIn(expected_unit_link, html)

    def test_public_container_preview_html(self):
        """
        Verify that a public xblock's container preview returns the expected HTML.
        """
        self.validate_preview_html(self.vertical, self.container_view,
                                   can_edit=False, can_reorder=False, can_add=False)
        self.validate_preview_html(self.child_container, self.container_view,
                                   can_edit=False, can_reorder=False, can_add=False)
        self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
                                   can_edit=False, can_reorder=False, can_add=False)

    def test_draft_container_preview_html(self):
        """
        Verify that a draft xblock's container preview returns the expected HTML.
        """
        draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
        draft_child_container = modulestore('draft').convert_to_draft(self.child_container.location)
        draft_child_vertical = modulestore('draft').convert_to_draft(self.child_vertical.location)
        self.validate_preview_html(draft_unit, self.container_view,
                                   can_edit=True, can_reorder=True, can_add=True)
        self.validate_preview_html(draft_child_container, self.container_view,
                                   can_edit=True, can_reorder=True, can_add=True)
        self.validate_preview_html(draft_child_vertical, self.reorderable_child_view,
                                   can_edit=True, can_reorder=True, can_add=True)

    def test_public_child_container_preview_html(self):
        """
        Verify that a public container rendered as a child of the container page returns the expected HTML.
        """
        empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                   category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=empty_child_container.location,
                           category='html', display_name='Split Child')
        self.validate_preview_html(empty_child_container, self.reorderable_child_view,
                                   can_reorder=False, can_edit=False, can_add=False)

    def test_draft_child_container_preview_html(self):
        """
        Verify that a draft container rendered as a child of the container page returns the expected HTML.
        """
        empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                   category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=empty_child_container.location,
                           category='html', display_name='Split Child')
        modulestore('draft').convert_to_draft(self.vertical.location)
        draft_empty_child_container = modulestore('draft').convert_to_draft(empty_child_container.location)
        self.validate_preview_html(draft_empty_child_container, self.reorderable_child_view,
                                   can_reorder=True, can_edit=False, can_add=False)

########NEW FILE########
__FILENAME__ = test_course_index
"""
Unit tests for getting the list of courses and the course outline.
"""
import json
import lxml

from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.locator import Locator


class TestCourseIndex(CourseTestCase):
    """
    Unit tests for getting the list of courses and the course outline.
    """
    def setUp(self):
        """
        Add a course with odd characters in the fields
        """
        super(TestCourseIndex, self).setUp()
        # had a problem where index showed course but has_access failed to retrieve it for non-staff
        self.odd_course = CourseFactory.create(
            org='test.org_1-2',
            number='test-2.3_course',
            display_name='dotted.course.name-2',
        )

    def check_index_and_outline(self, authed_client):
        """
        Test getting the list of courses and then pulling up their outlines
        """
        index_url = '/course/'
        index_response = authed_client.get(index_url, {}, HTTP_ACCEPT='text/html')
        parsed_html = lxml.html.fromstring(index_response.content)
        course_link_eles = parsed_html.find_class('course-link')
        self.assertGreaterEqual(len(course_link_eles), 2)
        for link in course_link_eles:
            self.assertRegexpMatches(
                link.get("href"),
                'course/slashes:{0}'.format(Locator.ALLOWED_ID_CHARS)
            )
            # now test that url
            outline_response = authed_client.get(link.get("href"), {}, HTTP_ACCEPT='text/html')
            # ensure it has the expected 2 self referential links
            outline_parsed = lxml.html.fromstring(outline_response.content)
            outline_link = outline_parsed.find_class('course-link')[0]
            self.assertEqual(outline_link.get("href"), link.get("href"))
            course_menu_link = outline_parsed.find_class('nav-course-courseware-outline')[0]
            self.assertEqual(course_menu_link.find("a").get("href"), link.get("href"))

    def test_is_staff_access(self):
        """
        Test that people with is_staff see the courses and can navigate into them
        """
        self.check_index_and_outline(self.client)

    def test_negative_conditions(self):
        """
        Test the error conditions for the access
        """
        outline_url = reverse_course_url('course_handler', self.course.id)
        # register a non-staff member and try to delete the course branch
        non_staff_client, _ = self.create_non_staff_authed_user_client()
        response = non_staff_client.delete(outline_url, {}, HTTP_ACCEPT='application/json')
        self.assertEqual(response.status_code, 403)

    def test_course_staff_access(self):
        """
        Make and register course_staff and ensure they can access the courses
        """
        course_staff_client, course_staff = self.create_non_staff_authed_user_client()
        for course in [self.course, self.odd_course]:
            permission_url = reverse_course_url('course_team_handler', course.id, kwargs={'email': course_staff.email})

            self.client.post(
                permission_url,
                data=json.dumps({"role": "staff"}),
                content_type="application/json",
                HTTP_ACCEPT="application/json",
            )

        # test access
        self.check_index_and_outline(course_staff_client)

    def test_json_responses(self):
        outline_url = reverse_course_url('course_handler', self.course.id)
        chapter = ItemFactory.create(parent_location=self.course.location, category='chapter', display_name="Week 1")
        lesson = ItemFactory.create(parent_location=chapter.location, category='sequential', display_name="Lesson 1")
        subsection = ItemFactory.create(parent_location=lesson.location, category='vertical', display_name='Subsection 1')
        ItemFactory.create(parent_location=subsection.location, category="video", display_name="My Video")

        resp = self.client.get(outline_url, HTTP_ACCEPT='application/json')
        json_response = json.loads(resp.content)

        # First spot check some values in the root response
        self.assertEqual(json_response['category'], 'course')
        self.assertEqual(json_response['id'], 'location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')
        self.assertEqual(json_response['display_name'], 'Robot Super Course')
        self.assertTrue(json_response['is_container'])
        self.assertFalse(json_response['is_draft'])

        # Now verify the first child
        children = json_response['children']
        self.assertTrue(len(children) > 0)
        first_child_response = children[0]
        self.assertEqual(first_child_response['category'], 'chapter')
        self.assertEqual(first_child_response['id'], 'location:MITx+999+Robot_Super_Course+chapter+Week_1')
        self.assertEqual(first_child_response['display_name'], 'Week 1')
        self.assertTrue(first_child_response['is_container'])
        self.assertFalse(first_child_response['is_draft'])
        self.assertTrue(len(first_child_response['children']) > 0)

        # Finally, validate the entire response for consistency
        self.assert_correct_json_response(json_response)

    def assert_correct_json_response(self, json_response):
        """
        Asserts that the JSON response is syntactically consistent
        """
        self.assertIsNotNone(json_response['display_name'])
        self.assertIsNotNone(json_response['id'])
        self.assertIsNotNone(json_response['category'])
        self.assertIsNotNone(json_response['is_draft'])
        self.assertIsNotNone(json_response['is_container'])
        if json_response['is_container']:
            for child_response in json_response['children']:
                self.assert_correct_json_response(child_response)
        else:
            self.assertFalse('children' in json_response)

########NEW FILE########
__FILENAME__ = test_course_updates
"""
unit tests for course_info views and models.
"""
import json

from contentstore.tests.test_course_settings import CourseTestCase
from contentstore.utils import reverse_course_url, reverse_usage_url
from xmodule.modulestore.locations import Location, SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore


class CourseUpdateTest(CourseTestCase):

    def create_update_url(self, provided_id=None, course_key=None):
        if course_key is None:
            course_key = self.course.id
        kwargs = {'provided_id': str(provided_id)} if provided_id else None
        return reverse_course_url('course_info_update_handler', course_key, kwargs=kwargs)

    '''The do all and end all of unit test cases.'''
    def test_course_update(self):
        '''Go through each interface and ensure it works.'''
        def get_response(content, date):
            """
            Helper method for making call to server and returning response.

            Does not supply a provided_id.
            """
            payload = {'content': content, 'date': date}
            url = self.create_update_url()

            resp = self.client.ajax_post(url, payload)
            self.assertContains(resp, '', status_code=200)

            return json.loads(resp.content)

        resp = self.client.get_html(
            reverse_course_url('course_info_handler', self.course.id)
        )
        self.assertContains(resp, 'Course Updates', status_code=200)

        init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
        content = init_content + '</iframe>'
        payload = get_response(content, 'January 8, 2013')
        self.assertHTMLEqual(payload['content'], content)

        first_update_url = self.create_update_url(provided_id=payload['id'])
        content += '<div>div <p>p<br/></p></div>'
        payload['content'] = content
        # POST requests were coming in w/ these header values causing an error; so, repro error here
        resp = self.client.ajax_post(
            first_update_url, payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
        )

        self.assertHTMLEqual(content, json.loads(resp.content)['content'],
                             "iframe w/ div")
        # refetch using provided id
        refetched = self.client.get_json(first_update_url)
        self.assertHTMLEqual(
            content, json.loads(refetched.content)['content'], "get w/ provided id"
        )

        # now put in an evil update
        content = '<ol/>'
        payload = get_response(content, 'January 11, 2013')
        self.assertHTMLEqual(content, payload['content'], "self closing ol")

        course_update_url = self.create_update_url()
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        self.assertTrue(len(payload) == 2)

        # try json w/o required fields
        self.assertContains(
            self.client.ajax_post(course_update_url, {'garbage': 1}),
            'Failed to save', status_code=400
        )

        # test an update with text in the tail of the header
        content = 'outside <strong>inside</strong> after'
        payload = get_response(content, 'June 22, 2000')
        self.assertHTMLEqual(content, payload['content'], "text outside tag")

        # now try to update a non-existent update
        content = 'blah blah'
        payload = {'content': content, 'date': 'January 21, 2013'}
        self.assertContains(
            self.client.ajax_post(course_update_url + '9', payload),
            'Failed to save', status_code=400
        )

        # update w/ malformed html
        content = '<garbage tag No closing brace to force <span>error</span>'
        payload = {'content': content,
                   'date': 'January 11, 2013'}

        self.assertContains(
            self.client.ajax_post(course_update_url, payload),
            '<garbage'
        )

        # set to valid html which would break an xml parser
        content = "<p><br><br></p>"
        payload = get_response(content, 'January 11, 2013')
        self.assertHTMLEqual(content, payload['content'])

        # now try to delete a non-existent update
        self.assertContains(self.client.delete(course_update_url + '19'), "delete", status_code=400)

        # now delete a real update
        content = 'blah blah'
        payload = get_response(content, 'January 28, 2013')
        this_id = payload['id']
        self.assertHTMLEqual(content, payload['content'], "single iframe")
        # first count the entries
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        before_delete = len(payload)

        url = self.create_update_url(provided_id=this_id)
        resp = self.client.delete(url)
        payload = json.loads(resp.content)
        self.assertTrue(len(payload) == before_delete - 1)

    def test_course_updates_compatibility(self):
        '''
        Test that course updates doesn't break on old data (content in 'data' field).
        Note: new data will save as list in 'items' field.
        '''
        # get the updates and populate 'data' field with some data.
        location = self.course.id.make_usage_key('course_info', 'updates')
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)
        update_date = u"January 23, 2014"
        update_content = u"Hello world!"
        update_data = u"<ol><li><h2>" + update_date + "</h2>" + update_content + "</li></ol>"
        course_updates.data = update_data
        modulestore('direct').update_item(course_updates, self.user)

        # test getting all updates list
        course_update_url = self.create_update_url()
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        self.assertEqual(payload, [{u'date': update_date, u'content': update_content, u'id': 1}])
        self.assertTrue(len(payload) == 1)

        # test getting single update item

        first_update_url = self.create_update_url(provided_id=payload[0]['id'])
        resp = self.client.get_json(first_update_url)
        payload = json.loads(resp.content)
        self.assertEqual(payload, {u'date': u'January 23, 2014', u'content': u'Hello world!', u'id': 1})
        self.assertHTMLEqual(update_date, payload['date'])
        self.assertHTMLEqual(update_content, payload['content'])

        # test that while updating it converts old data (with string format in 'data' field)
        # to new data (with list format in 'items' field) and respectively updates 'data' field.
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items, [])
        # now try to update first update item
        update_content = 'Testing'
        payload = {'content': update_content, 'date': update_date}
        resp = self.client.ajax_post(
            course_update_url + '1', payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
        )
        self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
        # course_updates 'data' field should update accordingly
        update_data = u"<section><article><h2>{date}</h2>{content}</article></section>".format(date=update_date, content=update_content)
        self.assertEqual(course_updates.data, update_data)

        # test delete course update item (soft delete)
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
        # now try to delete first update item
        resp = self.client.delete(course_update_url + '1')
        self.assertEqual(json.loads(resp.content), [])
        # confirm that course update is soft deleted ('status' flag set to 'deleted') in db
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items,
                         [{u'date': update_date, u'content': update_content, u'id': 1, u'status': 'deleted'}])

        # now try to get deleted update
        resp = self.client.get_json(course_update_url + '1')
        payload = json.loads(resp.content)
        self.assertEqual(payload.get('error'), u"Course update not found.")
        self.assertEqual(resp.status_code, 404)

        # now check that course update don't munges html
        update_content = u"""&lt;problem>
                           &lt;p>&lt;/p>
                           &lt;multiplechoiceresponse>
                           <pre>&lt;problem>
                               &lt;p>&lt;/p></pre>
                           <div><foo>bar</foo></div>"""
        payload = {'content': update_content, 'date': update_date}
        resp = self.client.ajax_post(
            course_update_url, payload, REQUEST_METHOD="POST"
        )
        self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])

    def test_no_ol_course_update(self):
        '''Test trying to add to a saved course_update which is not an ol.'''
        # get the updates and set to something wrong
        location = self.course.id.make_usage_key('course_info', 'updates')
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)
        course_updates.data = 'bad news'
        modulestore('direct').update_item(course_updates, self.user.id)

        init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
        content = init_content + '</iframe>'
        payload = {'content': content, 'date': 'January 8, 2013'}

        course_update_url = self.create_update_url()
        resp = self.client.ajax_post(course_update_url, payload)

        payload = json.loads(resp.content)

        self.assertHTMLEqual(payload['content'], content)

        # now confirm that the bad news and the iframe make up single update
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        self.assertTrue(len(payload) == 1)

    def test_post_course_update(self):
        """
        Test that a user can successfully post on course updates and handouts of a course
        """
        course_key = SlashSeparatedCourseKey('Org1', 'Course_1', 'Run_1')
        course_update_url = self.create_update_url(course_key=course_key)

        # create a course via the view handler
        self.client.ajax_post(course_update_url)

        block = u'updates'
        content = u"Sample update"
        payload = {'content': content, 'date': 'January 8, 2013'}
        resp = self.client.ajax_post(course_update_url, payload)

        # check that response status is 200 not 400
        self.assertEqual(resp.status_code, 200)

        payload = json.loads(resp.content)
        self.assertHTMLEqual(payload['content'], content)

        updates_location = self.course.id.make_usage_key('course_info', 'updates')
        self.assertTrue(isinstance(updates_location, Location))
        self.assertEqual(updates_location.name, block)

        # check posting on handouts
        handouts_location = self.course.id.make_usage_key('course_info', 'handouts')
        course_handouts_url = reverse_usage_url('xblock_handler', handouts_location)

        content = u"Sample handout"
        payload = {'data': content}
        resp = self.client.ajax_post(course_handouts_url, payload)
        # check that response status is 200 not 500
        self.assertEqual(resp.status_code, 200)

        payload = json.loads(resp.content)
        self.assertHTMLEqual(payload['data'], content)

########NEW FILE########
__FILENAME__ = test_helpers
"""
Unit tests for helpers.py.
"""

from contentstore.tests.utils import CourseTestCase
from contentstore.views.helpers import xblock_studio_url
from xmodule.modulestore.tests.factories import ItemFactory


class HelpersTestCase(CourseTestCase):
    """
    Unit tests for helpers.py.
    """
    def test_xblock_studio_url(self):

        # Verify course URL
        self.assertEqual(xblock_studio_url(self.course),
                         u'/course/slashes:MITx+999+Robot_Super_Course')

        # Verify chapter URL
        chapter = ItemFactory.create(parent_location=self.course.location, category='chapter',
                                     display_name="Week 1")
        self.assertIsNone(xblock_studio_url(chapter))

        # Verify lesson URL
        sequential = ItemFactory.create(parent_location=chapter.location, category='sequential',
                                        display_name="Lesson 1")
        self.assertIsNone(xblock_studio_url(sequential))

        # Verify vertical URL
        vertical = ItemFactory.create(parent_location=sequential.location, category='vertical',
                                      display_name='Unit')
        self.assertEqual(xblock_studio_url(vertical),
                         u'/unit/location:MITx+999+Robot_Super_Course+vertical+Unit')

        # Verify child vertical URL
        child_vertical = ItemFactory.create(parent_location=vertical.location, category='vertical',
                                            display_name='Child Vertical')
        self.assertEqual(xblock_studio_url(child_vertical),
                         u'/container/location:MITx+999+Robot_Super_Course+vertical+Child_Vertical')

        # Verify video URL
        video = ItemFactory.create(parent_location=child_vertical.location, category="video",
                                   display_name="My Video")
        self.assertIsNone(xblock_studio_url(video))

########NEW FILE########
__FILENAME__ = test_import_export
"""
Unit tests for course import and export
"""
import copy
import json
import logging
import os
import shutil
import tarfile
import tempfile
from path import path
from pymongo import MongoClient
from uuid import uuid4

from django.test.utils import override_settings
from django.conf import settings
from contentstore.utils import reverse_course_url

from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.factories import ItemFactory

from contentstore.tests.utils import CourseTestCase
from student import auth
from student.roles import CourseInstructorRole, CourseStaffRole

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex

log = logging.getLogger(__name__)


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ImportTestCase(CourseTestCase):
    """
    Unit tests for importing a course
    """
    def setUp(self):
        super(ImportTestCase, self).setUp()
        self.url = reverse_course_url('import_handler', self.course.id)
        self.content_dir = path(tempfile.mkdtemp())

        def touch(name):
            """ Equivalent to shell's 'touch'"""
            with file(name, 'a'):
                os.utime(name, None)

        # Create tar test files -----------------------------------------------
        # OK course:
        good_dir = tempfile.mkdtemp(dir=self.content_dir)
        os.makedirs(os.path.join(good_dir, "course"))
        with open(os.path.join(good_dir, "course.xml"), "w+") as f:
            f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')

        with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f:
            f.write('<course></course>')

        self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
        with tarfile.open(self.good_tar, "w:gz") as gtar:
            gtar.add(good_dir)

        # Bad course (no 'course.xml' file):
        bad_dir = tempfile.mkdtemp(dir=self.content_dir)
        touch(os.path.join(bad_dir, "bad.xml"))
        self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
        with tarfile.open(self.bad_tar, "w:gz") as btar:
            btar.add(bad_dir)

        self.unsafe_common_dir = path(tempfile.mkdtemp(dir=self.content_dir))

    def tearDown(self):
        shutil.rmtree(self.content_dir)
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_no_coursexml(self):
        """
        Check that the response for a tar.gz import without a course.xml is
        correct.
        """
        with open(self.bad_tar) as btar:
            resp = self.client.post(
                self.url,
                {
                    "name": self.bad_tar,
                    "course-data": [btar]
                })
        self.assertEquals(resp.status_code, 415)
        # Check that `import_status` returns the appropriate stage (i.e., the
        # stage at which import failed).
        resp_status = self.client.get(
            reverse_course_url(
                'import_status_handler',
                self.course.id,
                kwargs={'filename': os.path.split(self.bad_tar)[1]}
            )
        )

        self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2)

    def test_with_coursexml(self):
        """
        Check that the response for a tar.gz import with a course.xml is
        correct.
        """
        with open(self.good_tar) as gtar:
            args = {"name": self.good_tar, "course-data": [gtar]}
            resp = self.client.post(self.url, args)

        self.assertEquals(resp.status_code, 200)

    def test_import_in_existing_course(self):
        """
        Check that course is imported successfully in existing course and users have their access roles
        """
        # Create a non_staff user and add it to course staff only
        __, nonstaff_user = self.create_non_staff_authed_user_client(authenticate=False)
        auth.add_users(self.user, CourseStaffRole(self.course.id), nonstaff_user)

        course = self.store.get_course(self.course.id)
        self.assertIsNotNone(course)
        display_name_before_import = course.display_name

        # Check that global staff user can import course
        with open(self.good_tar) as gtar:
            args = {"name": self.good_tar, "course-data": [gtar]}
            resp = self.client.post(self.url, args)
        self.assertEquals(resp.status_code, 200)

        course = self.store.get_course(self.course.id)
        self.assertIsNotNone(course)
        display_name_after_import = course.display_name

        # Check that course display name have changed after import
        self.assertNotEqual(display_name_before_import, display_name_after_import)

        # Now check that non_staff user has his same role
        self.assertFalse(CourseInstructorRole(self.course.id).has_user(nonstaff_user))
        self.assertTrue(CourseStaffRole(self.course.id).has_user(nonstaff_user))

        # Now course staff user can also successfully import course
        self.client.login(username=nonstaff_user.username, password='foo')
        with open(self.good_tar) as gtar:
            args = {"name": self.good_tar, "course-data": [gtar]}
            resp = self.client.post(self.url, args)
        self.assertEquals(resp.status_code, 200)

        # Now check that non_staff user has his same role
        self.assertFalse(CourseInstructorRole(self.course.id).has_user(nonstaff_user))
        self.assertTrue(CourseStaffRole(self.course.id).has_user(nonstaff_user))

    ## Unsafe tar methods #####################################################
    # Each of these methods creates a tarfile with a single type of unsafe
    # content.
    def _fifo_tar(self):
        """
        Tar file with FIFO
        """
        fifop = self.unsafe_common_dir / "fifo.file"
        fifo_tar = self.unsafe_common_dir / "fifo.tar.gz"
        os.mkfifo(fifop)
        with tarfile.open(fifo_tar, "w:gz") as tar:
            tar.add(fifop)

        return fifo_tar

    def _symlink_tar(self):
        """
        Tarfile with symlink to path outside directory.
        """
        outsidep = self.unsafe_common_dir / "unsafe_file.txt"
        symlinkp = self.unsafe_common_dir / "symlink.txt"
        symlink_tar = self.unsafe_common_dir / "symlink.tar.gz"
        outsidep.symlink(symlinkp)
        with tarfile.open(symlink_tar, "w:gz") as tar:
            tar.add(symlinkp)

        return symlink_tar

    def _outside_tar(self):
        """
        Tarfile with file that extracts to outside directory.

        Extracting this tarfile in directory <dir> will put its contents
        directly in <dir> (rather than <dir/tarname>).
        """
        outside_tar = self.unsafe_common_dir / "unsafe_file.tar.gz"
        with tarfile.open(outside_tar, "w:gz") as tar:
            tar.addfile(tarfile.TarInfo(str(self.content_dir / "a_file")))

        return outside_tar

    def _outside_tar2(self):
        """
        Tarfile with file that extracts to outside directory.

        The path here matches the basename (`self.unsafe_common_dir`), but
        then "cd's out". E.g. "/usr/../etc" == "/etc", but the naive basename
        of the first (but not the second) is "/usr"

        Extracting this tarfile in directory <dir> will also put its contents
        directly in <dir> (rather than <dir/tarname>).
        """
        outside_tar = self.unsafe_common_dir / "unsafe_file.tar.gz"
        with tarfile.open(outside_tar, "w:gz") as tar:
            tar.addfile(tarfile.TarInfo(str(self.unsafe_common_dir / "../a_file")))

        return outside_tar

    def test_unsafe_tar(self):
        """
        Check that safety measure work.

        This includes:
            'tarbombs' which include files or symlinks with paths
        outside or directly in the working directory,
            'special files' (character device, block device or FIFOs),

        all raise exceptions/400s.
        """

        def try_tar(tarpath):
            with open(tarpath) as tar:
                args = {"name": tarpath, "course-data": [tar]}
                resp = self.client.post(self.url, args)
            self.assertEquals(resp.status_code, 400)
            self.assertTrue("SuspiciousFileOperation" in resp.content)

        try_tar(self._fifo_tar())
        try_tar(self._symlink_tar())
        try_tar(self._outside_tar())
        try_tar(self._outside_tar2())
        # Check that `import_status` returns the appropriate stage (i.e.,
        # either 3, indicating all previous steps are completed, or 0,
        # indicating no upload in progress)
        resp_status = self.client.get(
            reverse_course_url(
                'import_status_handler',
                self.course.id,
                kwargs={'filename': os.path.split(self.good_tar)[1]}
            )
        )
        import_status = json.loads(resp_status.content)["ImportStatus"]
        self.assertIn(import_status, (0, 3))


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ExportTestCase(CourseTestCase):
    """
    Tests for export_handler.
    """
    def setUp(self):
        """
        Sets up the test course.
        """
        super(ExportTestCase, self).setUp()
        self.url = reverse_course_url('export_handler', self.course.id)

    def test_export_html(self):
        """
        Get the HTML for the page.
        """
        resp = self.client.get_html(self.url)
        self.assertEquals(resp.status_code, 200)
        self.assertContains(resp, "Export My Course Content")

    def test_export_json_unsupported(self):
        """
        JSON is unsupported.
        """
        resp = self.client.get(self.url, HTTP_ACCEPT='application/json')
        self.assertEquals(resp.status_code, 406)

    def test_export_targz(self):
        """
        Get tar.gz file, using HTTP_ACCEPT.
        """
        resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz')
        self._verify_export_succeeded(resp)

    def test_export_targz_urlparam(self):
        """
        Get tar.gz file, using URL parameter.
        """
        resp = self.client.get(self.url + '?_accept=application/x-tgz')
        self._verify_export_succeeded(resp)

    def _verify_export_succeeded(self, resp):
        """ Export success helper method. """
        self.assertEquals(resp.status_code, 200)
        self.assertTrue(resp.get('Content-Disposition').startswith('attachment'))

    def test_export_failure_top_level(self):
        """
        Export failure.
        """
        ItemFactory.create(parent_location=self.course.location, category='aawefawef')
        self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')

    def test_export_failure_subsection_level(self):
        """
        Slightly different export failure.
        """
        vertical = ItemFactory.create(parent_location=self.course.location, category='vertical', display_name='foo')
        ItemFactory.create(
            parent_location=vertical.location,
            category='aawefawef'
        )

        self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+vertical+foo')

    def _verify_export_failure(self, expectedText):
        """ Export failure helper method. """
        resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz')
        self.assertEquals(resp.status_code, 200)
        self.assertIsNone(resp.get('Content-Disposition'))
        self.assertContains(resp, 'Unable to create xml for module')
        self.assertContains(resp, expectedText)

########NEW FILE########
__FILENAME__ = test_item
"""Tests for items views."""

import json
from datetime import datetime
import ddt

from mock import patch
from pytz import UTC
from webob import Response

from django.http import Http404
from django.test import TestCase
from django.test.client import RequestFactory
from django.core.urlresolvers import reverse
from contentstore.utils import reverse_usage_url

from contentstore.views.component import component_handler

from contentstore.tests.utils import CourseTestCase
from contentstore.utils import compute_publish_state, PublishState
from student.tests.factories import UserFactory
from xmodule.capa_module import CapaDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.keys import UsageKey
from xmodule.modulestore.locations import Location


class ItemTest(CourseTestCase):
    """ Base test class for create, save, and delete """
    def setUp(self):
        super(ItemTest, self).setUp()

        self.course_key = self.course.id
        self.usage_key = self.course.location

    @staticmethod
    def get_item_from_modulestore(usage_key, draft=False):
        """
        Get the item referenced by the UsageKey from the modulestore
        """
        store = modulestore('draft') if draft else modulestore('direct')
        return store.get_item(usage_key)

    def response_usage_key(self, response):
        """
        Get the UsageKey from the response payload and verify that the status_code was 200.
        :param response:
        """
        parsed = json.loads(response.content)
        self.assertEqual(response.status_code, 200)
        return UsageKey.from_string(parsed['locator'])

    def create_xblock(self, parent_usage_key=None, display_name=None, category=None, boilerplate=None):
        data = {
            'parent_locator': unicode(self.usage_key) if parent_usage_key is None else unicode(parent_usage_key),
            'category': category
        }
        if display_name is not None:
            data['display_name'] = display_name
        if boilerplate is not None:
            data['boilerplate'] = boilerplate
        return self.client.ajax_post(reverse('contentstore.views.xblock_handler'), json.dumps(data))


class GetItem(ItemTest):
    """Tests for '/xblock' GET url."""

    def _create_vertical(self, parent_usage_key=None):
        """
        Creates a vertical, returning its UsageKey.
        """
        resp = self.create_xblock(category='vertical', parent_usage_key=parent_usage_key)
        self.assertEqual(resp.status_code, 200)
        return self.response_usage_key(resp)

    def _get_container_preview(self, usage_key):
        """
        Returns the HTML and resources required for the xblock at the specified UsageKey
        """
        preview_url = reverse_usage_url("xblock_view_handler", usage_key, {'view_name': 'container_preview'})
        resp = self.client.get(preview_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)
        resp_content = json.loads(resp.content)
        html = resp_content['html']
        self.assertTrue(html)
        resources = resp_content['resources']
        self.assertIsNotNone(resources)
        return html, resources

    def test_get_vertical(self):
        # Add a vertical
        resp = self.create_xblock(category='vertical')
        usage_key = self.response_usage_key(resp)

        # Retrieve it
        resp = self.client.get(reverse_usage_url('xblock_handler', usage_key))
        self.assertEqual(resp.status_code, 200)

    def test_get_empty_container_fragment(self):
        root_usage_key = self._create_vertical()
        html, __ = self._get_container_preview(root_usage_key)

        # Verify that the Studio wrapper is not added
        self.assertNotIn('wrapper-xblock', html)

        # Verify that the header and article tags are still added
        self.assertIn('<header class="xblock-header">', html)
        self.assertIn('<article class="xblock-render">', html)

    def test_get_container_fragment(self):
        root_usage_key = self._create_vertical()

        # Add a problem beneath a child vertical
        child_vertical_usage_key = self._create_vertical(parent_usage_key=root_usage_key)
        resp = self.create_xblock(parent_usage_key=child_vertical_usage_key, category='problem', boilerplate='multiplechoice.yaml')
        self.assertEqual(resp.status_code, 200)

        # Get the preview HTML
        html, __ = self._get_container_preview(root_usage_key)

        # Verify that the Studio nesting wrapper has been added
        self.assertIn('level-nesting', html)
        self.assertIn('<header class="xblock-header">', html)
        self.assertIn('<article class="xblock-render">', html)

        # Verify that the Studio element wrapper has been added
        self.assertIn('level-element', html)

    def test_get_container_nested_container_fragment(self):
        """
        Test the case of the container page containing a link to another container page.
        """
        # Add a wrapper with child beneath a child vertical
        root_usage_key = self._create_vertical()

        resp = self.create_xblock(parent_usage_key=root_usage_key, category="wrapper")
        self.assertEqual(resp.status_code, 200)
        wrapper_usage_key = self.response_usage_key(resp)

        resp = self.create_xblock(parent_usage_key=wrapper_usage_key, category='problem', boilerplate='multiplechoice.yaml')
        self.assertEqual(resp.status_code, 200)

        # Get the preview HTML and verify the View -> link is present.
        html, __ = self._get_container_preview(root_usage_key)
        self.assertIn('wrapper-xblock', html)
        self.assertRegexpMatches(
            html,
            # The instance of the wrapper class will have an auto-generated ID. Allow any
            # characters after wrapper.
            (r'"/container/location:MITx\+999\+Robot_Super_Course\+wrapper\+\w+" class="action-button">\s*'
             '<span class="action-button-text">View</span>')
        )

    def test_split_test(self):
        """
        Test that a split_test module renders all of its children in Studio.
        """
        root_usage_key = self._create_vertical()
        resp = self.create_xblock(category='split_test', parent_usage_key=root_usage_key)
        split_test_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=split_test_usage_key, category='html', boilerplate='announcement.yaml')
        self.assertEqual(resp.status_code, 200)
        resp = self.create_xblock(parent_usage_key=split_test_usage_key, category='html', boilerplate='zooming_image.yaml')
        self.assertEqual(resp.status_code, 200)
        html, __ = self._get_container_preview(split_test_usage_key)
        self.assertIn('Announcement', html)
        self.assertIn('Zooming', html)


class DeleteItem(ItemTest):
    """Tests for '/xblock' DELETE url."""
    def test_delete_static_page(self):
        # Add static tab
        resp = self.create_xblock(category='static_tab')
        usage_key = self.response_usage_key(resp)

        # Now delete it. There was a bug that the delete was failing (static tabs do not exist in draft modulestore).
        resp = self.client.delete(reverse_usage_url('xblock_handler', usage_key))
        self.assertEqual(resp.status_code, 204)


class TestCreateItem(ItemTest):
    """
    Test the create_item handler thoroughly
    """
    def test_create_nicely(self):
        """
        Try the straightforward use cases
        """
        # create a chapter
        display_name = 'Nicely created'
        resp = self.create_xblock(display_name=display_name, category='chapter')

        # get the new item and check its category and display_name
        chap_usage_key = self.response_usage_key(resp)
        new_obj = self.get_item_from_modulestore(chap_usage_key)
        self.assertEqual(new_obj.scope_ids.block_type, 'chapter')
        self.assertEqual(new_obj.display_name, display_name)
        self.assertEqual(new_obj.location.org, self.course.location.org)
        self.assertEqual(new_obj.location.course, self.course.location.course)

        # get the course and ensure it now points to this one
        course = self.get_item_from_modulestore(self.usage_key)
        self.assertIn(chap_usage_key, course.children)

        # use default display name
        resp = self.create_xblock(parent_usage_key=chap_usage_key, category='vertical')
        vert_usage_key = self.response_usage_key(resp)

        # create problem w/ boilerplate
        template_id = 'multiplechoice.yaml'
        resp = self.create_xblock(
            parent_usage_key=vert_usage_key,
            category='problem',
            boilerplate=template_id
        )
        prob_usage_key = self.response_usage_key(resp)
        problem = self.get_item_from_modulestore(prob_usage_key, True)
        # ensure it's draft
        self.assertTrue(problem.is_draft)
        # check against the template
        template = CapaDescriptor.get_template(template_id)
        self.assertEqual(problem.data, template['data'])
        self.assertEqual(problem.display_name, template['metadata']['display_name'])
        self.assertEqual(problem.markdown, template['metadata']['markdown'])

    def test_create_item_negative(self):
        """
        Negative tests for create_item
        """
        # non-existent boilerplate: creates a default
        resp = self.create_xblock(category='problem', boilerplate='nosuchboilerplate.yaml')
        self.assertEqual(resp.status_code, 200)

    def test_create_with_future_date(self):
        self.assertEqual(self.course.start, datetime(2030, 1, 1, tzinfo=UTC))
        resp = self.create_xblock(category='chapter')
        usage_key = self.response_usage_key(resp)
        obj = self.get_item_from_modulestore(usage_key)
        self.assertEqual(obj.start, datetime(2030, 1, 1, tzinfo=UTC))


class TestDuplicateItem(ItemTest):
    """
    Test the duplicate method.
    """
    def setUp(self):
        """ Creates the test course structure and a few components to 'duplicate'. """
        super(TestDuplicateItem, self).setUp()
        # Create a parent chapter (for testing children of children).
        resp = self.create_xblock(parent_usage_key=self.usage_key, category='chapter')
        self.chapter_usage_key = self.response_usage_key(resp)

        # create a sequential containing a problem and an html component
        resp = self.create_xblock(parent_usage_key=self.chapter_usage_key, category='sequential')
        self.seq_usage_key = self.response_usage_key(resp)

        # create problem and an html component
        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='problem', boilerplate='multiplechoice.yaml')
        self.problem_usage_key = self.response_usage_key(resp)

        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='html')
        self.html_usage_key = self.response_usage_key(resp)

        # Create a second sequential just (testing children of children)
        self.create_xblock(parent_usage_key=self.chapter_usage_key, category='sequential2')

    def test_duplicate_equality(self):
        """
        Tests that a duplicated xblock is identical to the original,
        except for location and display name.
        """
        def duplicate_and_verify(source_usage_key, parent_usage_key):
            usage_key = self._duplicate_item(parent_usage_key, source_usage_key)
            self.assertTrue(check_equality(source_usage_key, usage_key), "Duplicated item differs from original")

        def check_equality(source_usage_key, duplicate_usage_key):
            original_item = self.get_item_from_modulestore(source_usage_key, draft=True)
            duplicated_item = self.get_item_from_modulestore(duplicate_usage_key, draft=True)

            self.assertNotEqual(
                original_item.location,
                duplicated_item.location,
                "Location of duplicate should be different from original"
            )
            # Set the location and display name to be the same so we can make sure the rest of the duplicate is equal.
            duplicated_item.location = original_item.location
            duplicated_item.display_name = original_item.display_name

            # Children will also be duplicated, so for the purposes of testing equality, we will set
            # the children to the original after recursively checking the children.
            if original_item.has_children:
                self.assertEqual(
                    len(original_item.children),
                    len(duplicated_item.children),
                    "Duplicated item differs in number of children"
                )
                for i in xrange(len(original_item.children)):
                    if not check_equality(original_item.children[i], duplicated_item.children[i]):
                        return False
                duplicated_item.children = original_item.children

            return original_item == duplicated_item

        duplicate_and_verify(self.problem_usage_key, self.seq_usage_key)
        duplicate_and_verify(self.html_usage_key, self.seq_usage_key)
        duplicate_and_verify(self.seq_usage_key, self.chapter_usage_key)
        duplicate_and_verify(self.chapter_usage_key, self.usage_key)

    def test_ordering(self):
        """
        Tests the a duplicated xblock appears immediately after its source
        (if duplicate and source share the same parent), else at the
        end of the children of the parent.
        """
        def verify_order(source_usage_key, parent_usage_key, source_position=None):
            usage_key = self._duplicate_item(parent_usage_key, source_usage_key)
            parent = self.get_item_from_modulestore(parent_usage_key)
            children = parent.children
            if source_position is None:
                self.assertFalse(source_usage_key in children, 'source item not expected in children array')
                self.assertEqual(
                    children[len(children) - 1],
                    usage_key,
                    "duplicated item not at end"
                )
            else:
                self.assertEqual(
                    children[source_position],
                    source_usage_key,
                    "source item at wrong position"
                )
                self.assertEqual(
                    children[source_position + 1],
                    usage_key,
                    "duplicated item not ordered after source item"
                )

        verify_order(self.problem_usage_key, self.seq_usage_key, 0)
        # 2 because duplicate of problem should be located before.
        verify_order(self.html_usage_key, self.seq_usage_key, 2)
        verify_order(self.seq_usage_key, self.chapter_usage_key, 0)

        # Test duplicating something into a location that is not the parent of the original item.
        # Duplicated item should appear at the end.
        verify_order(self.html_usage_key, self.usage_key)

    def test_display_name(self):
        """
        Tests the expected display name for the duplicated xblock.
        """
        def verify_name(source_usage_key, parent_usage_key, expected_name, display_name=None):
            usage_key = self._duplicate_item(parent_usage_key, source_usage_key, display_name)
            duplicated_item = self.get_item_from_modulestore(usage_key, draft=True)
            self.assertEqual(duplicated_item.display_name, expected_name)
            return usage_key

        # Display name comes from template.
        dupe_usage_key = verify_name(self.problem_usage_key, self.seq_usage_key, "Duplicate of 'Multiple Choice'")
        # Test dupe of dupe.
        verify_name(dupe_usage_key, self.seq_usage_key, "Duplicate of 'Duplicate of 'Multiple Choice''")

        # Uses default display_name of 'Text' from HTML component.
        verify_name(self.html_usage_key, self.seq_usage_key, "Duplicate of 'Text'")

        # The sequence does not have a display_name set, so category is shown.
        verify_name(self.seq_usage_key, self.chapter_usage_key, "Duplicate of sequential")

        # Now send a custom display name for the duplicate.
        verify_name(self.seq_usage_key, self.chapter_usage_key, "customized name", display_name="customized name")

    def _duplicate_item(self, parent_usage_key, source_usage_key, display_name=None):
        data = {
            'parent_locator': unicode(parent_usage_key),
            'duplicate_source_locator': unicode(source_usage_key)
        }
        if display_name is not None:
            data['display_name'] = display_name

        resp = self.client.ajax_post(reverse('contentstore.views.xblock_handler'), json.dumps(data))
        return self.response_usage_key(resp)


class TestEditItem(ItemTest):
    """
    Test xblock update.
    """
    def setUp(self):
        """ Creates the test course structure and a couple problems to 'edit'. """
        super(TestEditItem, self).setUp()
        # create a chapter
        display_name = 'chapter created'
        resp = self.create_xblock(display_name=display_name, category='chapter')
        chap_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=chap_usage_key, category='sequential')
        self.seq_usage_key = self.response_usage_key(resp)
        self.seq_update_url = reverse_usage_url("xblock_handler", self.seq_usage_key)

        # create problem w/ boilerplate
        template_id = 'multiplechoice.yaml'
        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='problem', boilerplate=template_id)
        self.problem_usage_key = self.response_usage_key(resp)
        self.problem_update_url = reverse_usage_url("xblock_handler", self.problem_usage_key)

        self.course_update_url = reverse_usage_url("xblock_handler", self.usage_key)

    def test_delete_field(self):
        """
        Sending null in for a field 'deletes' it
        """
        self.client.ajax_post(
            self.problem_update_url,
            data={'metadata': {'rerandomize': 'onreset'}}
        )
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(problem.rerandomize, 'onreset')
        self.client.ajax_post(
            self.problem_update_url,
            data={'metadata': {'rerandomize': None}}
        )
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(problem.rerandomize, 'never')

    def test_null_field(self):
        """
        Sending null in for a field 'deletes' it
        """
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(problem.markdown)
        self.client.ajax_post(
            self.problem_update_url,
            data={'nullout': ['markdown']}
        )
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNone(problem.markdown)

    def test_date_fields(self):
        """
        Test setting due & start dates on sequential
        """
        sequential = self.get_item_from_modulestore(self.seq_usage_key)
        self.assertIsNone(sequential.due)
        self.client.ajax_post(
            self.seq_update_url,
            data={'metadata': {'due': '2010-11-22T04:00Z'}}
        )
        sequential = self.get_item_from_modulestore(self.seq_usage_key)
        self.assertEqual(sequential.due, datetime(2010, 11, 22, 4, 0, tzinfo=UTC))
        self.client.ajax_post(
            self.seq_update_url,
            data={'metadata': {'start': '2010-09-12T14:00Z'}}
        )
        sequential = self.get_item_from_modulestore(self.seq_usage_key)
        self.assertEqual(sequential.due, datetime(2010, 11, 22, 4, 0, tzinfo=UTC))
        self.assertEqual(sequential.start, datetime(2010, 9, 12, 14, 0, tzinfo=UTC))

    def test_delete_child(self):
        """
        Test deleting a child.
        """
        # Create 2 children of main course.
        resp_1 = self.create_xblock(display_name='child 1', category='chapter')
        resp_2 = self.create_xblock(display_name='child 2', category='chapter')
        chapter1_usage_key = self.response_usage_key(resp_1)
        chapter2_usage_key = self.response_usage_key(resp_2)

        course = self.get_item_from_modulestore(self.usage_key)
        self.assertIn(chapter1_usage_key, course.children)
        self.assertIn(chapter2_usage_key, course.children)

        # Remove one child from the course.
        resp = self.client.ajax_post(
            self.course_update_url,
            data={'children': [unicode(chapter2_usage_key)]}
        )
        self.assertEqual(resp.status_code, 200)

        # Verify that the child is removed.
        course = self.get_item_from_modulestore(self.usage_key)
        self.assertNotIn(chapter1_usage_key, course.children)
        self.assertIn(chapter2_usage_key, course.children)

    def test_reorder_children(self):
        """
        Test reordering children that can be in the draft store.
        """
        # Create 2 child units and re-order them. There was a bug about @draft getting added
        # to the IDs.
        unit_1_resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='vertical')
        unit_2_resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='vertical')
        unit1_usage_key = self.response_usage_key(unit_1_resp)
        unit2_usage_key = self.response_usage_key(unit_2_resp)

        # The sequential already has a child defined in the setUp (a problem).
        # Children must be on the sequential to reproduce the original bug,
        # as it is important that the parent (sequential) NOT be in the draft store.
        children = self.get_item_from_modulestore(self.seq_usage_key).children
        self.assertEqual(unit1_usage_key, children[1])
        self.assertEqual(unit2_usage_key, children[2])

        resp = self.client.ajax_post(
            self.seq_update_url,
            data={'children': [unicode(self.problem_usage_key), unicode(unit2_usage_key), unicode(unit1_usage_key)]}
        )
        self.assertEqual(resp.status_code, 200)

        children = self.get_item_from_modulestore(self.seq_usage_key).children
        self.assertEqual(self.problem_usage_key, children[0])
        self.assertEqual(unit1_usage_key, children[2])
        self.assertEqual(unit2_usage_key, children[1])

    def test_make_public(self):
        """ Test making a private problem public (publishing it). """
        # When the problem is first created, it is only in draft (because of its category).
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))

    def test_make_private(self):
        """ Test making a public problem private (un-publishing it). """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it private
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_private'}
        )
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)

    def test_make_draft(self):
        """ Test creating a draft version of a public problem. """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it draft, which means both versions will exist.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'create_draft'}
        )
        # Update the draft version and check that published is different.
        self.client.ajax_post(
            self.problem_update_url,
            data={'metadata': {'due': '2077-10-10T04:00Z'}}
        )
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        self.assertIsNone(published.due)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_make_public_with_update(self):
        """ Update a problem and make it public at the same time. """
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'metadata': {'due': '2077-10-10T04:00Z'},
                'publish': 'make_public'
            }
        )
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        self.assertEqual(published.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_make_private_with_update(self):
        """ Make a problem private and update it at the same time. """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'metadata': {'due': '2077-10-10T04:00Z'},
                'publish': 'make_private'
            }
        )
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_create_draft_with_update(self):
        """ Create a draft and update it at the same time. """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it draft, which means both versions will exist.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'metadata': {'due': '2077-10-10T04:00Z'},
                'publish': 'create_draft'
            }
        )
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        self.assertIsNone(published.due)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_create_draft_with_multiple_requests(self):
        """
        Create a draft request returns already created version if it exists.
        """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it draft, which means both versions will exist.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'create_draft'
            }
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        draft_1 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_1)

        # Now check that when a user sends request to create a draft when there is already a draft version then
        # user gets that already created draft instead of getting 'DuplicateItemError' exception.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'create_draft'
            }
        )
        draft_2 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_2)
        self.assertEqual(draft_1, draft_2)


    def test_make_private_with_multiple_requests(self):
        """
        Make private requests gets proper response even if xmodule is already made private.
        """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))

        # Now make it private, and check that its published version not exists
        resp = self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'make_private'
            }
        )
        self.assertEqual(resp.status_code, 200)
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        draft_1 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_1)

        # Now check that when a user sends request to make it private when it already is private then
        # user gets that private version instead of getting 'ItemNotFoundError' exception.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'make_private'
            }
        )
        self.assertEqual(resp.status_code, 200)
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        draft_2 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_2)
        self.assertEqual(draft_1, draft_2)


    def test_published_and_draft_contents_with_update(self):
        """ Create a draft and publish it then modify the draft and check that published content is not modified """

        # Make problem public.
        resp = self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))

        # Now make a draft
        resp = self.client.ajax_post(
            self.problem_update_url,
            data={
                'id': unicode(self.problem_usage_key),
                'metadata': {},
                'data': "<p>Problem content draft.</p>",
                'publish': 'create_draft'
            }
        )

        # Both published and draft content should be different
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertNotEqual(draft.data, published.data)

        # Get problem by 'xblock_handler'
        view_url = reverse_usage_url("xblock_view_handler", self.problem_usage_key, {"view_name": "student_view"})
        resp = self.client.get(view_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)

        # Activate the editing view
        view_url = reverse_usage_url("xblock_view_handler", self.problem_usage_key, {"view_name": "studio_view"})
        resp = self.client.get(view_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)

        # Both published and draft content should still be different
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertNotEqual(draft.data, published.data)

    def test_publish_states_of_nested_xblocks(self):
        """ Test publishing of a unit page containing a nested xblock  """

        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, display_name='Test Unit', category='vertical')
        unit_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=unit_usage_key, category='wrapper')
        wrapper_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=wrapper_usage_key, category='html')
        html_usage_key = self.response_usage_key(resp)

        # The unit and its children should be private initially
        unit_update_url = reverse_usage_url('xblock_handler', unit_usage_key)
        unit = self.get_item_from_modulestore(unit_usage_key, True)
        html = self.get_item_from_modulestore(html_usage_key, True)
        self.assertEqual(compute_publish_state(unit), PublishState.private)
        self.assertEqual(compute_publish_state(html), PublishState.private)

        # Make the unit public and verify that the problem is also made public
        resp = self.client.ajax_post(
            unit_update_url,
            data={'publish': 'make_public'}
        )
        self.assertEqual(resp.status_code, 200)
        unit = self.get_item_from_modulestore(unit_usage_key, True)
        html = self.get_item_from_modulestore(html_usage_key, True)
        self.assertEqual(compute_publish_state(unit), PublishState.public)
        self.assertEqual(compute_publish_state(html), PublishState.public)

        # Make a draft for the unit and verify that the problem also has a draft
        resp = self.client.ajax_post(
            unit_update_url,
            data={
                'id': unicode(unit_usage_key),
                'metadata': {},
                'publish': 'create_draft'
            }
        )
        self.assertEqual(resp.status_code, 200)
        unit = self.get_item_from_modulestore(unit_usage_key, True)
        html = self.get_item_from_modulestore(html_usage_key, True)
        self.assertEqual(compute_publish_state(unit), PublishState.draft)
        self.assertEqual(compute_publish_state(html), PublishState.draft)


@ddt.ddt
class TestComponentHandler(TestCase):
    def setUp(self):
        self.request_factory = RequestFactory()

        patcher = patch('contentstore.views.component.get_modulestore')
        self.get_modulestore = patcher.start()
        self.addCleanup(patcher.stop)

        self.descriptor = self.get_modulestore.return_value.get_item.return_value

        self.usage_key_string = unicode(
            Location('dummy_org', 'dummy_course', 'dummy_run', 'dummy_category', 'dummy_name')
        )

        self.user = UserFactory()

        self.request = self.request_factory.get('/dummy-url')
        self.request.user = self.user

    def test_invalid_handler(self):
        self.descriptor.handle.side_effect = Http404

        with self.assertRaises(Http404):
            component_handler(self.request, self.usage_key_string, 'invalid_handler')

    @ddt.data('GET', 'POST', 'PUT', 'DELETE')
    def test_request_method(self, method):

        def check_handler(handler, request, suffix):
            self.assertEquals(request.method, method)
            return Response()

        self.descriptor.handle = check_handler

        # Have to use the right method to create the request to get the HTTP method that we want
        req_factory_method = getattr(self.request_factory, method.lower())
        request = req_factory_method('/dummy-url')
        request.user = self.user

        component_handler(request, self.usage_key_string, 'dummy_handler')

    @ddt.data(200, 404, 500)
    def test_response_code(self, status_code):
        def create_response(handler, request, suffix):
            return Response(status_code=status_code)

        self.descriptor.handle = create_response

        self.assertEquals(component_handler(self.request, self.usage_key_string, 'dummy_handler').status_code, status_code)

########NEW FILE########
__FILENAME__ = test_preview
"""
Tests for contentstore.views.preview.py
"""
from django.test import TestCase
from django.test.client import RequestFactory

from student.tests.factories import UserFactory

from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory

from contentstore.views.preview import get_preview_fragment


class GetPreviewHtmlTestCase(TestCase):
    """
    Tests for get_preview_fragment.

    Note that there are other existing test cases in test_contentstore that indirectly execute
    get_preview_fragment via the xblock RESTful API.
    """

    def test_preview_fragment(self):
        """
        Test for calling get_preview_html.

        This test used to be specifically about Locators (ensuring that they did not
        get translated to Locations). The test now has questionable value.
        """
        course = CourseFactory.create()
        html = ItemFactory.create(
            parent_location=course.location,
            category="html",
            data={'data': "<html>foobar</html>"}
        )

        request = RequestFactory().get('/dummy-url')
        request.user = UserFactory()
        request.session = {}

        # Call get_preview_fragment directly.
        html = get_preview_fragment(request, html, {}).content

        # Verify student view html is returned, and the usage ID is as expected.
        self.assertRegexpMatches(
            html,
            'data-usage-id="location:MITx\+999\+Robot_Super_Course\+html\+html_[0-9]*"'
        )
        self.assertRegexpMatches(html, '<html>foobar</html>')

########NEW FILE########
__FILENAME__ = test_tabs
""" Tests for tab functions (just primitive). """

import json
from contentstore.views import tabs
from contentstore.tests.utils import CourseTestCase
from django.test import TestCase
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.tabs import CourseTabList, WikiTab
from contentstore.utils import reverse_course_url
from xmodule.modulestore.django import modulestore


class TabsPageTests(CourseTestCase):
    """Test cases for Tabs (a.k.a Pages) page"""

    def setUp(self):
        """Common setup for tests"""

        # call super class to setup course, etc.
        super(TabsPageTests, self).setUp()

        # Set the URL for tests
        self.url = reverse_course_url('tabs_handler', self.course.id)

        # add a static tab to the course, for code coverage
        self.test_tab = ItemFactory.create(
            parent_location=self.course.location,
            category="static_tab",
            display_name="Static_1"
        )
        self.reload_course()

    def check_invalid_tab_id_response(self, resp):
        """Verify response is an error listing the invalid_tab_id"""

        self.assertEqual(resp.status_code, 400)
        resp_content = json.loads(resp.content)
        self.assertIn("error", resp_content)
        self.assertIn("invalid_tab_id", resp_content['error'])

    def test_not_implemented(self):
        """Verify not implemented errors"""

        # JSON GET request not supported
        with self.assertRaises(NotImplementedError):
            self.client.get(self.url)

        # JSON POST request not supported
        with self.assertRaises(NotImplementedError):
            self.client.ajax_post(
                self.url,
                data=json.dumps({
                    'tab_id_locator': {'tab_id': WikiTab.type},
                    'unsupported_request': None,
                }),
            )

        # invalid JSON POST request
        with self.assertRaises(NotImplementedError):
            self.client.ajax_post(
                self.url,
                data={'invalid_request': None},
            )

    def test_view_index(self):
        """Basic check that the Pages page responds correctly"""

        resp = self.client.get_html(self.url)
        self.assertEqual(resp.status_code, 200)
        self.assertIn('course-nav-list', resp.content)

    def test_reorder_tabs(self):
        """Test re-ordering of tabs"""

        # get the original tab ids
        orig_tab_ids = [tab.tab_id for tab in self.course.tabs]
        tab_ids = list(orig_tab_ids)
        num_orig_tabs = len(orig_tab_ids)

        # make sure we have enough tabs to play around with
        self.assertTrue(num_orig_tabs >= 5)

        # reorder the last two tabs
        tab_ids[num_orig_tabs - 1], tab_ids[num_orig_tabs - 2] = tab_ids[num_orig_tabs - 2], tab_ids[num_orig_tabs - 1]

        # remove the middle tab
        # (the code needs to handle the case where tabs requested for re-ordering is a subset of the tabs in the course)
        removed_tab = tab_ids.pop(num_orig_tabs / 2)
        self.assertTrue(len(tab_ids) == num_orig_tabs - 1)

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data={'tabs': [{'tab_id': tab_id} for tab_id in tab_ids]},
        )
        self.assertEqual(resp.status_code, 204)

        # reload the course and verify the new tab order
        self.reload_course()
        new_tab_ids = [tab.tab_id for tab in self.course.tabs]
        self.assertEqual(new_tab_ids, tab_ids + [removed_tab])
        self.assertNotEqual(new_tab_ids, orig_tab_ids)

    def test_reorder_tabs_invalid_list(self):
        """Test re-ordering of tabs with invalid tab list"""

        orig_tab_ids = [tab.tab_id for tab in self.course.tabs]
        tab_ids = list(orig_tab_ids)

        # reorder the first two tabs
        tab_ids[0], tab_ids[1] = tab_ids[1], tab_ids[0]

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data={'tabs': [{'tab_id': tab_id} for tab_id in tab_ids]},
        )
        self.assertEqual(resp.status_code, 400)
        resp_content = json.loads(resp.content)
        self.assertIn("error", resp_content)

    def test_reorder_tabs_invalid_tab(self):
        """Test re-ordering of tabs with invalid tab"""

        invalid_tab_ids = ['courseware', 'info', 'invalid_tab_id']

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data={'tabs': [{'tab_id': tab_id} for tab_id in invalid_tab_ids]},
        )
        self.check_invalid_tab_id_response(resp)

    def check_toggle_tab_visiblity(self, tab_type, new_is_hidden_setting):
        """Helper method to check changes in tab visibility"""

        # find the tab
        old_tab = CourseTabList.get_tab_by_type(self.course.tabs, tab_type)

        # visibility should be different from new setting
        self.assertNotEqual(old_tab.is_hidden, new_is_hidden_setting)

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data=json.dumps({
                'tab_id_locator': {'tab_id': old_tab.tab_id},
                'is_hidden': new_is_hidden_setting,
            }),
        )
        self.assertEqual(resp.status_code, 204)

        # reload the course and verify the new visibility setting
        self.reload_course()
        new_tab = CourseTabList.get_tab_by_type(self.course.tabs, tab_type)
        self.assertEqual(new_tab.is_hidden, new_is_hidden_setting)

    def test_toggle_tab_visibility(self):
        """Test toggling of tab visiblity"""

        self.check_toggle_tab_visiblity(WikiTab.type, True)
        self.check_toggle_tab_visiblity(WikiTab.type, False)

    def test_toggle_invalid_tab_visibility(self):
        """Test toggling visibility of an invalid tab"""

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data=json.dumps({
                'tab_id_locator': {'tab_id': 'invalid_tab_id'}
            }),
        )
        self.check_invalid_tab_id_response(resp)

    def test_tab_preview_html(self):
        """
        Verify that the static tab renders itself with the correct HTML
        """
        preview_url = '/xblock/{}/student_view'.format(self.test_tab.location)

        resp = self.client.get(preview_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)
        resp_content = json.loads(resp.content)
        html = resp_content['html']

        # Verify that the HTML contains the expected elements
        self.assertIn('<span class="action-button-text">Edit</span>', html)
        self.assertIn('<span class="sr">Duplicate this component</span>', html)
        self.assertIn('<span class="sr">Delete this component</span>', html)
        self.assertIn('<span data-tooltip="Drag to reorder" class="drag-handle action"></span>', html)



class PrimitiveTabEdit(TestCase):
    """Tests for the primitive tab edit data manipulations"""

    def test_delete(self):
        """Test primitive tab deletion."""
        course = CourseFactory.create(org='edX', course='999')
        with self.assertRaises(ValueError):
            tabs.primitive_delete(course, 0)
        with self.assertRaises(ValueError):
            tabs.primitive_delete(course, 1)
        with self.assertRaises(IndexError):
            tabs.primitive_delete(course, 6)
        tabs.primitive_delete(course, 2)
        self.assertFalse({u'type': u'textbooks'} in course.tabs)
        # Check that discussion has shifted up
        self.assertEquals(course.tabs[2], {'type': 'discussion', 'name': 'Discussion'})

    def test_insert(self):
        """Test primitive tab insertion."""
        course = CourseFactory.create(org='edX', course='999')
        tabs.primitive_insert(course, 2, 'notes', 'aname')
        self.assertEquals(course.tabs[2], {'type': 'notes', 'name': 'aname'})
        with self.assertRaises(ValueError):
            tabs.primitive_insert(course, 0, 'notes', 'aname')
        with self.assertRaises(ValueError):
            tabs.primitive_insert(course, 3, 'static_tab', 'aname')

    def test_save(self):
        """Test course saving."""
        course = CourseFactory.create(org='edX', course='999')
        tabs.primitive_insert(course, 3, 'notes', 'aname')
        course2 = modulestore().get_course(course.id)
        self.assertEquals(course2.tabs[3], {'type': 'notes', 'name': 'aname'})

########NEW FILE########
__FILENAME__ = test_textbooks
import json
from unittest import TestCase
from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url

from contentstore.views.course import (
    validate_textbooks_json, validate_textbook_json, TextbookValidationError)


class TextbookIndexTestCase(CourseTestCase):
    "Test cases for the textbook index page"
    def setUp(self):
        "Set the URL for tests"
        super(TextbookIndexTestCase, self).setUp()
        self.url = reverse_course_url('textbooks_list_handler', self.course.id)

    def test_view_index(self):
        "Basic check that the textbook index page responds correctly"
        resp = self.client.get(self.url)
        self.assertEqual(resp.status_code, 200)
        # we don't have resp.context right now,
        # due to bugs in our testing harness :(
        if resp.context:
            self.assertEqual(resp.context['course'], self.course)

    def test_view_index_xhr(self):
        "Check that we get a JSON response when requested via AJAX"
        resp = self.client.get(
            self.url,
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(self.course.pdf_textbooks, obj)

    def test_view_index_xhr_content(self):
        "Check that the response maps to the content of the modulestore"
        content = [
            {
                "tab_title": "my textbook",
                "url": "/abc.pdf",
                "id": "992"
            }, {
                "tab_title": "pineapple",
                "id": "0pineapple",
                "chapters": [
                    {
                        "title": "The Fruit",
                        "url": "/a/b/fruit.pdf",
                    }, {
                        "title": "The Legend",
                        "url": "/b/c/legend.pdf",
                    }
                ]
            }
        ]
        self.course.pdf_textbooks = content
        self.save_course()

        resp = self.client.get(
            self.url,
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(content, obj)

    def test_view_index_xhr_put(self):
        "Check that you can save information to the server"
        textbooks = [
            {"tab_title": "Hi, mom!"},
            {"tab_title": "Textbook 2"},
        ]
        resp = self.client.put(
            self.url,
            data=json.dumps(textbooks),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 200)

        # should be the same, except for added ID
        no_ids = []
        self.reload_course()
        for textbook in self.course.pdf_textbooks:
            del textbook["id"]
            no_ids.append(textbook)
        self.assertEqual(no_ids, textbooks)

    def test_view_index_xhr_put_invalid(self):
        "Check that you can't save invalid JSON"
        resp = self.client.put(
            self.url,
            data="invalid",
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 400)
        obj = json.loads(resp.content)
        self.assertIn("error", obj)


class TextbookCreateTestCase(CourseTestCase):
    "Test cases for creating a new PDF textbook"

    def setUp(self):
        "Set up a url and some textbook content for tests"
        super(TextbookCreateTestCase, self).setUp()
        self.url = reverse_course_url('textbooks_list_handler', self.course.id)

        self.textbook = {
            "tab_title": "Economics",
            "chapters": {
                "title": "Chapter 1",
                "url": "/a/b/c/ch1.pdf",
            }
        }

    def test_happy_path(self):
        "Test that you can create a textbook"
        resp = self.client.post(
            self.url,
            data=json.dumps(self.textbook),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH="XMLHttpRequest",
        )
        self.assertEqual(resp.status_code, 201)
        self.assertIn("Location", resp)
        textbook = json.loads(resp.content)
        self.assertIn("id", textbook)
        del textbook["id"]
        self.assertEqual(self.textbook, textbook)

    def test_valid_id(self):
        "Textbook IDs must begin with a number; try a valid one"
        self.textbook["id"] = "7x5"
        resp = self.client.post(
            self.url,
            data=json.dumps(self.textbook),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH="XMLHttpRequest",
        )
        self.assertEqual(resp.status_code, 201)
        textbook = json.loads(resp.content)
        self.assertEqual(self.textbook, textbook)

    def test_invalid_id(self):
        "Textbook IDs must begin with a number; try an invalid one"
        self.textbook["id"] = "xxx"
        resp = self.client.post(
            self.url,
            data=json.dumps(self.textbook),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH="XMLHttpRequest",
        )
        self.assertEqual(resp.status_code, 400)
        self.assertNotIn("Location", resp)


class TextbookDetailTestCase(CourseTestCase):
    "Test cases for the `textbook_detail_handler` view"

    def setUp(self):
        "Set some useful content and URLs for tests"
        super(TextbookDetailTestCase, self).setUp()
        self.textbook1 = {
            "tab_title": "Economics",
            "id": 1,
            "chapters": {
                "title": "Chapter 1",
                "url": "/a/b/c/ch1.pdf",
            }
        }
        self.url1 = self.get_details_url("1")

        self.textbook2 = {
            "tab_title": "Algebra",
            "id": 2,
            "chapters": {
                "title": "Chapter 11",
                "url": "/a/b/ch11.pdf",
            }
        }
        self.url2 = self.get_details_url("2")
        self.course.pdf_textbooks = [self.textbook1, self.textbook2]
        # Save the data that we've just changed to the underlying
        # MongoKeyValueStore before we update the mongo datastore.
        self.save_course()
        self.url_nonexist = self.get_details_url("1=20")

    def get_details_url(self, textbook_id):
        """
        Returns the URL for textbook detail handler.
        """
        return reverse_course_url(
            'textbooks_detail_handler',
            self.course.id,
            kwargs={'textbook_id': textbook_id}
        )

    def test_get_1(self):
        "Get the first textbook"
        resp = self.client.get(self.url1)
        self.assertEqual(resp.status_code, 200)
        compare = json.loads(resp.content)
        self.assertEqual(compare, self.textbook1)

    def test_get_2(self):
        "Get the second textbook"
        resp = self.client.get(self.url2)
        self.assertEqual(resp.status_code, 200)
        compare = json.loads(resp.content)
        self.assertEqual(compare, self.textbook2)

    def test_get_nonexistant(self):
        "Get a nonexistent textbook"
        resp = self.client.get(self.url_nonexist)
        self.assertEqual(resp.status_code, 404)

    def test_delete(self):
        "Delete a textbook by ID"
        resp = self.client.delete(self.url1)
        self.assertEqual(resp.status_code, 204)
        self.reload_course()
        self.assertEqual(self.course.pdf_textbooks, [self.textbook2])

    def test_delete_nonexistant(self):
        "Delete a textbook by ID, when the ID doesn't match an existing textbook"
        resp = self.client.delete(self.url_nonexist)
        self.assertEqual(resp.status_code, 404)
        self.reload_course()
        self.assertEqual(self.course.pdf_textbooks, [self.textbook1, self.textbook2])

    def test_create_new_by_id(self):
        "Create a textbook by ID"
        textbook = {
            "tab_title": "a new textbook",
            "url": "supercool.pdf",
            "id": "1supercool",
        }
        url = self.get_details_url("1supercool")
        resp = self.client.post(
            url,
            data=json.dumps(textbook),
            content_type="application/json",
        )
        self.assertEqual(resp.status_code, 201)
        resp2 = self.client.get(url)
        self.assertEqual(resp2.status_code, 200)
        compare = json.loads(resp2.content)
        self.assertEqual(compare, textbook)
        self.reload_course()
        self.assertEqual(
            self.course.pdf_textbooks,
            [self.textbook1, self.textbook2, textbook]
        )

    def test_replace_by_id(self):
        "Create a textbook by ID, overwriting an existing textbook ID"
        replacement = {
            "tab_title": "You've been replaced!",
            "url": "supercool.pdf",
            "id": "2",
        }
        resp = self.client.post(
            self.url2,
            data=json.dumps(replacement),
            content_type="application/json",
        )
        self.assertEqual(resp.status_code, 201)
        resp2 = self.client.get(self.url2)
        self.assertEqual(resp2.status_code, 200)
        compare = json.loads(resp2.content)
        self.assertEqual(compare, replacement)
        course = self.store.get_item(self.course.location)
        self.assertEqual(
            course.pdf_textbooks,
            [self.textbook1, replacement]
        )


class TextbookValidationTestCase(TestCase):
    "Tests for the code to validate the structure of a PDF textbook"

    def setUp(self):
        "Set some useful content for tests"
        self.tb1 = {
            "tab_title": "Hi, mom!",
            "url": "/mom.pdf"
        }
        self.tb2 = {
            "tab_title": "Hi, dad!",
            "chapters": [
                {
                    "title": "Baseball",
                    "url": "baseball.pdf",
                }, {
                    "title": "Basketball",
                    "url": "crazypants.pdf",
                }
            ]
        }
        self.textbooks = [self.tb1, self.tb2]

    def test_happy_path_plural(self):
        "Test that the plural validator works properly"
        result = validate_textbooks_json(json.dumps(self.textbooks))
        self.assertEqual(self.textbooks, result)

    def test_happy_path_singular_1(self):
        "Test that the singular validator works properly"
        result = validate_textbook_json(json.dumps(self.tb1))
        self.assertEqual(self.tb1, result)

    def test_happy_path_singular_2(self):
        "Test that the singular validator works properly, with different data"
        result = validate_textbook_json(json.dumps(self.tb2))
        self.assertEqual(self.tb2, result)

    def test_valid_id(self):
        "Test that a valid ID doesn't trip the validator, and comes out unchanged"
        self.tb1["id"] = 1
        result = validate_textbook_json(json.dumps(self.tb1))
        self.assertEqual(self.tb1, result)

    def test_invalid_id(self):
        "Test that an invalid ID trips the validator"
        self.tb1["id"] = "abc"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json(json.dumps(self.tb1))

    def test_invalid_json_plural(self):
        "Test that invalid JSON trips the plural validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json("[{'abc'}]")

    def test_invalid_json_singular(self):
        "Test that invalid JSON trips the singluar validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json("[{1]}")

    def test_wrong_json_plural(self):
        "Test that a JSON object trips the plural validators (requires a list)"
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json('{"tab_title": "Hi, mom!"}')

    def test_wrong_json_singular(self):
        "Test that a JSON list trips the plural validators (requires an object)"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json('[{"tab_title": "Hi, mom!"}, {"tab_title": "Hi, dad!"}]')

    def test_no_tab_title_plural(self):
        "Test that `tab_title` is required for the plural validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json('[{"url": "/textbook.pdf"}]')

    def test_no_tab_title_singular(self):
        "Test that `tab_title` is required for the singular validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json('{"url": "/textbook.pdf"}')

    def test_duplicate_ids(self):
        "Test that duplicate IDs in the plural validator trips the validator"
        textbooks = [{
            "tab_title": "name one",
            "url": "one.pdf",
            "id": 1,
        }, {
            "tab_title": "name two",
            "url": "two.pdf",
            "id": 1,
        }]
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json(json.dumps(textbooks))

########NEW FILE########
__FILENAME__ = test_transcripts
"""Tests for items views."""

import os
import json
import tempfile
from uuid import uuid4
import copy
import textwrap
from pymongo import MongoClient

from django.core.urlresolvers import reverse
from django.test.utils import override_settings
from django.conf import settings

from contentstore.tests.utils import CourseTestCase
from cache_toolbox.core import del_cached_content
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.keys import UsageKey
from xmodule.video_module import transcripts_utils

from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class Basetranscripts(CourseTestCase):
    """Base test class for transcripts tests."""

    def clear_subs_content(self):
        """Remove, if transcripts content exists."""
        for youtube_id in self.get_youtube_ids().values():
            filename = 'subs_{0}.srt.sjson'.format(youtube_id)
            content_location = StaticContent.compute_location(self.course.id, filename)
            try:
                content = contentstore().find(content_location)
                contentstore().delete(content.get_id())
            except NotFoundError:
                pass

    def setUp(self):
        """Create initial data."""
        super(Basetranscripts, self).setUp()

        # Add video module
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'video',
            'type': 'video'
        }
        resp = self.client.ajax_post('/xblock/', data)
        self.assertEqual(resp.status_code, 200)

        self.video_usage_key = self._get_usage_key(resp)
        self.item = modulestore().get_item(self.video_usage_key)
        # hI10vDNYz4M - valid Youtube ID with transcripts.
        # JMD_ifUUfsU, AKqURZnYqpk, DYpADpL7jAY - valid Youtube IDs without transcripts.
        self.item.data = '<video youtube="0.75:JMD_ifUUfsU,1.0:hI10vDNYz4M,1.25:AKqURZnYqpk,1.50:DYpADpL7jAY" />'
        modulestore().update_item(self.item, self.user.id)

        self.item = modulestore().get_item(self.video_usage_key)
        # Remove all transcripts for current module.
        self.clear_subs_content()

    def _get_usage_key(self, resp):
        """ Returns the usage key from the response returned by a create operation. """
        usage_key_string = json.loads(resp.content).get('locator')
        return UsageKey.from_string(usage_key_string)

    def get_youtube_ids(self):
        """Return youtube speeds and ids."""
        item = modulestore().get_item(self.video_usage_key)

        return {
            0.75: item.youtube_id_0_75,
            1: item.youtube_id_1_0,
            1.25: item.youtube_id_1_25,
            1.5: item.youtube_id_1_5
        }

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()


class TestUploadtranscripts(Basetranscripts):
    """Tests for '/transcripts/upload' url."""

    def setUp(self):
        """Create initial data."""
        super(TestUploadtranscripts, self).setUp()

        self.good_srt_file = tempfile.NamedTemporaryFile(suffix='.srt')
        self.good_srt_file.write(textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """))
        self.good_srt_file.seek(0)

        self.bad_data_srt_file = tempfile.NamedTemporaryFile(suffix='.srt')
        self.bad_data_srt_file.write('Some BAD data')
        self.bad_data_srt_file.seek(0)

        self.bad_name_srt_file = tempfile.NamedTemporaryFile(suffix='.BAD')
        self.bad_name_srt_file.write(textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """))
        self.bad_name_srt_file.seek(0)

    def test_success_video_module_source_subs_uploading(self):
        self.item.data = textwrap.dedent("""
            <video youtube="">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """)
        modulestore().update_item(self.item, self.user.id)

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': self.video_usage_key,
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(json.loads(resp.content).get('status'), 'Success')

        item = modulestore().get_item(self.video_usage_key)
        self.assertEqual(item.sub, filename)

        content_location = StaticContent.compute_location(
            self.course.id, 'subs_{0}.srt.sjson'.format(filename))
        self.assertTrue(contentstore().find(content_location))

    def test_fail_data_without_id(self):
        link = reverse('upload_transcripts')
        resp = self.client.post(link, {'transcript-file': self.good_srt_file})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'POST data without "locator" form data.')

    def test_fail_data_without_file(self):
        link = reverse('upload_transcripts')
        resp = self.client.post(link, {'locator': self.video_usage_key})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'POST data without "file" form data.')

    def test_fail_data_with_bad_locator(self):
        # Test for raising `InvalidLocationError` exception.
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': 'BAD_LOCATOR',
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

        # Test for raising `ItemNotFoundError` exception.
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': '{0}_{1}'.format(self.video_usage_key, 'BAD_LOCATOR'),
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

    def test_fail_for_non_video_module(self):
        # non_video module: setup
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'non_video',
            'type': 'non_video'
        }
        resp = self.client.ajax_post('/xblock/', data)
        usage_key = self._get_usage_key(resp)
        item = modulestore().get_item(usage_key)
        item.data = '<non_video youtube="0.75:JMD_ifUUfsU,1.0:hI10vDNYz4M" />'
        modulestore().update_item(item, self.user.id)

        # non_video module: testing

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(usage_key),
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Transcripts are supported only for "video" modules.')

    def test_fail_bad_xml(self):
        self.item.data = '<<<video youtube="0.75:JMD_ifUUfsU,1.25:AKqURZnYqpk,1.50:DYpADpL7jAY" />'
        modulestore().update_item(self.item, self.user.id)

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(self.video_usage_key),
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })

        self.assertEqual(resp.status_code, 400)
        # incorrect xml produces incorrect item category error
        self.assertEqual(json.loads(resp.content).get('status'), 'Transcripts are supported only for "video" modules.')

    def test_fail_bad_data_srt_file(self):
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.bad_data_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(self.video_usage_key),
            'transcript-file': self.bad_data_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Something wrong with SubRip transcripts file during parsing.')

    def test_fail_bad_name_srt_file(self):
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.bad_name_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(self.video_usage_key),
            'transcript-file': self.bad_name_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'We support only SubRip (*.srt) transcripts format.')

    def test_undefined_file_extension(self):
        srt_file = tempfile.NamedTemporaryFile(suffix='')
        srt_file.write(textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """))
        srt_file.seek(0)

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': self.video_usage_key,
            'transcript-file': srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Undefined file extension.')

    def tearDown(self):
        super(TestUploadtranscripts, self).tearDown()

        self.good_srt_file.close()
        self.bad_data_srt_file.close()
        self.bad_name_srt_file.close()


class TestDownloadtranscripts(Basetranscripts):
    """Tests for '/transcripts/download' url."""

    def save_subs_to_store(self, subs, subs_id):
        """Save transcripts into `StaticContent`."""
        filedata = json.dumps(subs, indent=2)
        mime_type = 'application/json'
        filename = 'subs_{0}.srt.sjson'.format(subs_id)

        content_location = StaticContent.compute_location(self.course.id, filename)
        content = StaticContent(content_location, filename, mime_type, filedata)
        contentstore().save(content)
        del_cached_content(content_location)
        return content_location

    def test_success_download_youtube(self):
        self.item.data = '<video youtube="1:JMD_ifUUfsU" />'
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, 'JMD_ifUUfsU')

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key, 'subs_id': "JMD_ifUUfsU"})
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(resp.content, """0\n00:00:00,100 --> 00:00:00,200\nsubs #1\n\n1\n00:00:00,200 --> 00:00:00,240\nsubs #2\n\n2\n00:00:00,240 --> 00:00:00,380\nsubs #3\n\n""")

    def test_success_download_nonyoutube(self):
        subs_id = str(uuid4())
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """.format(subs_id))
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key, 'subs_id': subs_id})
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(
            resp.content,
            '0\n00:00:00,100 --> 00:00:00,200\nsubs #1\n\n1\n00:00:00,200 --> '
            '00:00:00,240\nsubs #2\n\n2\n00:00:00,240 --> 00:00:00,380\nsubs #3\n\n'
        )
        transcripts_utils.remove_subs_from_store(subs_id, self.item)

    def test_fail_data_without_file(self):
        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': ''})
        self.assertEqual(resp.status_code, 404)

        resp = self.client.get(link, {})
        self.assertEqual(resp.status_code, 404)

    def test_fail_data_with_bad_locator(self):
        # Test for raising `InvalidLocationError` exception.
        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': 'BAD_LOCATOR'})
        self.assertEqual(resp.status_code, 404)

        # Test for raising `ItemNotFoundError` exception.
        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': '{0}_{1}'.format(self.video_usage_key, 'BAD_LOCATOR')})
        self.assertEqual(resp.status_code, 404)

    def test_fail_for_non_video_module(self):
        # Video module: setup
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'videoalpha',
            'type': 'videoalpha'
        }
        resp = self.client.ajax_post('/xblock/', data)
        usage_key = self._get_usage_key(resp)
        subs_id = str(uuid4())
        item = modulestore().get_item(usage_key)
        item.data = textwrap.dedent("""
            <videoalpha youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </videoalpha>
        """.format(subs_id))
        modulestore().update_item(item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': unicode(usage_key)})
        self.assertEqual(resp.status_code, 404)

    def test_fail_nonyoutube_subs_dont_exist(self):
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="UNDEFINED">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """)
        modulestore().update_item(self.item, self.user.id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key})
        self.assertEqual(resp.status_code, 404)

    def test_empty_youtube_attr_and_sub_attr(self):
        self.item.data = textwrap.dedent("""
            <video youtube="">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """)
        modulestore().update_item(self.item, self.user.id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key})

        self.assertEqual(resp.status_code, 404)

    def test_fail_bad_sjson_subs(self):
        subs_id = str(uuid4())
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """.format(subs_id))
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1'
            ]
        }
        self.save_subs_to_store(subs, 'JMD_ifUUfsU')

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key})

        self.assertEqual(resp.status_code, 404)


class TestChecktranscripts(Basetranscripts):
    """Tests for '/transcripts/check' url."""

    def save_subs_to_store(self, subs, subs_id):
        """Save transcripts into `StaticContent`."""
        filedata = json.dumps(subs, indent=2)
        mime_type = 'application/json'
        filename = 'subs_{0}.srt.sjson'.format(subs_id)

        content_location = StaticContent.compute_location(self.course.id, filename)
        content = StaticContent(content_location, filename, mime_type, filedata)
        contentstore().save(content)
        del_cached_content(content_location)
        return content_location

    def test_success_download_nonyoutube(self):
        subs_id = str(uuid4())
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """.format(subs_id))
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        data = {
            'locator': unicode(self.video_usage_key),
            'videos': [{
                'type': 'html5',
                'video': subs_id,
                'mode': 'mp4',
            }]
        }
        link = reverse('check_transcripts')
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 200)
        self.assertDictEqual(
            json.loads(resp.content),
            {
                u'status': u'Success',
                u'subs': unicode(subs_id),
                u'youtube_local': False,
                u'is_youtube_mode': False,
                u'youtube_server': False,
                u'command': u'found',
                u'current_item_subs': unicode(subs_id),
                u'youtube_diff': True,
                u'html5_local': [unicode(subs_id)],
                u'html5_equal': False,
            }
        )

        transcripts_utils.remove_subs_from_store(subs_id, self.item)

    def test_check_youtube(self):
        self.item.data = '<video youtube="1:JMD_ifUUfsU" />'
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, 'JMD_ifUUfsU')
        link = reverse('check_transcripts')
        data = {
            'locator': unicode(self.video_usage_key),
            'videos': [{
                'type': 'youtube',
                'video': 'JMD_ifUUfsU',
                'mode': 'youtube',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 200)
        self.assertDictEqual(
            json.loads(resp.content),
            {
                u'status': u'Success',
                u'subs': u'JMD_ifUUfsU',
                u'youtube_local': True,
                u'is_youtube_mode': True,
                u'youtube_server': False,
                u'command': u'found',
                u'current_item_subs': None,
                u'youtube_diff': True,
                u'html5_local': [],
                u'html5_equal': False,
            }
        )

    def test_fail_data_without_id(self):
        link = reverse('check_transcripts')
        data = {
            'locator': '',
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

    def test_fail_data_with_bad_locator(self):
        # Test for raising `InvalidLocationError` exception.
        link = reverse('check_transcripts')
        data = {
            'locator': '',
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

        # Test for raising `ItemNotFoundError` exception.
        data = {
            'locator': '{0}_{1}'.format(self.video_usage_key, 'BAD_LOCATOR'),
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

    def test_fail_for_non_video_module(self):
        # Not video module: setup
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'not_video',
            'type': 'not_video'
        }
        resp = self.client.ajax_post('/xblock/', data)
        usage_key = self._get_usage_key(resp)
        subs_id = str(uuid4())
        item = modulestore().get_item(usage_key)
        item.data = textwrap.dedent("""
            <not_video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </videoalpha>
        """.format(subs_id))
        modulestore().update_item(item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        data = {
            'locator': unicode(usage_key),
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        link = reverse('check_transcripts')
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Transcripts are supported only for "video" modules.')

########NEW FILE########
__FILENAME__ = test_unit_page
"""
Unit tests for the unit page.
"""

from contentstore.views.tests.utils import StudioPageTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import ItemFactory


class UnitPageTestCase(StudioPageTestCase):
    """
    Unit tests for the unit page.
    """

    def setUp(self):
        super(UnitPageTestCase, self).setUp()
        self.vertical = ItemFactory.create(parent_location=self.sequential.location,
                                           category='vertical', display_name='Unit')
        self.video = ItemFactory.create(parent_location=self.vertical.location,
                                        category="video", display_name="My Video")

    def test_public_unit_page_html(self):
        """
        Verify that an xblock returns the expected HTML for a public unit page.
        """
        html = self.get_page_html(self.vertical)
        self.validate_html_for_add_buttons(html)

    def test_draft_unit_page_html(self):
        """
        Verify that an xblock returns the expected HTML for a draft unit page.
        """
        draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
        html = self.get_page_html(draft_unit)
        self.validate_html_for_add_buttons(html)

    def test_public_component_preview_html(self):
        """
        Verify that a public xblock's preview returns the expected HTML.
        """
        self.validate_preview_html(self.video, 'student_view',
                                   can_edit=True, can_reorder=True, can_add=False)

    def test_draft_component_preview_html(self):
        """
        Verify that a draft xblock's preview returns the expected HTML.
        """
        modulestore('draft').convert_to_draft(self.vertical.location)
        draft_video = modulestore('draft').convert_to_draft(self.video.location)
        self.validate_preview_html(draft_video, 'student_view',
                                   can_edit=True, can_reorder=True, can_add=False)

    def test_public_child_container_preview_html(self):
        """
        Verify that a public child container rendering on the unit page (which shows a View arrow
        to the container page) returns the expected HTML.
        """
        child_container = ItemFactory.create(parent_location=self.vertical.location,
                                             category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=child_container.location,
                           category='html', display_name='grandchild')
        self.validate_preview_html(child_container, 'student_view',
                                   can_reorder=True, can_edit=False, can_add=False)

    def test_draft_child_container_preview_html(self):
        """
        Verify that a draft child container rendering on the unit page (which shows a View arrow
        to the container page) returns the expected HTML.
        """
        child_container = ItemFactory.create(parent_location=self.vertical.location,
                                             category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=child_container.location,
                           category='html', display_name='grandchild')
        modulestore('draft').convert_to_draft(self.vertical.location)
        draft_child_container = modulestore('draft').get_item(child_container.location)
        self.validate_preview_html(draft_child_container, 'student_view',
                                   can_reorder=True, can_edit=False, can_add=False)

########NEW FILE########
__FILENAME__ = test_user
"""
Tests for contentstore/views/user.py.
"""
import json

from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from django.contrib.auth.models import User
from student.models import CourseEnrollment
from student.roles import CourseStaffRole, CourseInstructorRole
from student import auth


class UsersTestCase(CourseTestCase):
    def setUp(self):
        super(UsersTestCase, self).setUp()
        self.ext_user = User.objects.create_user(
            "joe", "joe@comedycentral.com", "haha")
        self.ext_user.is_active = True
        self.ext_user.is_staff = False
        self.ext_user.save()
        self.inactive_user = User.objects.create_user(
            "carl", "carl@comedycentral.com", "haha")
        self.inactive_user.is_active = False
        self.inactive_user.is_staff = False
        self.inactive_user.save()

        self.index_url = self.course_team_url()
        self.detail_url = self.course_team_url(email=self.ext_user.email)
        self.inactive_detail_url = self.course_team_url(email=self.inactive_user.email)
        self.invalid_detail_url = self.course_team_url(email='nonexistent@user.com')

    def course_team_url(self, email=None):
        return reverse_course_url(
            'course_team_handler', self.course.id,
            kwargs={'email': email} if email else {}
        )

    def test_index(self):
        resp = self.client.get(self.index_url, HTTP_ACCEPT='text/html')
        # ext_user is not currently a member of the course team, and so should
        # not show up on the page.
        self.assertNotContains(resp, self.ext_user.email)

    def test_index_member(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.ext_user)

        resp = self.client.get(self.index_url, HTTP_ACCEPT='text/html')
        self.assertContains(resp, self.ext_user.email)

    def test_detail(self):
        resp = self.client.get(self.detail_url)
        self.assertEqual(resp.status_code, 200)
        result = json.loads(resp.content)
        self.assertEqual(result["role"], None)
        self.assertTrue(result["active"])

    def test_detail_inactive(self):
        resp = self.client.get(self.inactive_detail_url)
        self.assertEqual(resp.status_code, 200)
        result = json.loads(resp.content)
        self.assertFalse(result["active"])

    def test_detail_invalid(self):
        resp = self.client.get(self.invalid_detail_url)
        self.assertEqual(resp.status_code, 404)
        result = json.loads(resp.content)
        self.assertIn("error", result)

    def test_detail_post(self):
        resp = self.client.post(
            self.detail_url,
            data={"role": None},
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        # no content: should not be in any roles
        self.assertFalse(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_not_enrolled()

    def test_detail_post_staff(self):
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_enrolled()

    def test_detail_post_staff_other_inst(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.user)

        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_enrolled()
        # check that other user is unchanged
        user = User.objects.get(email=self.user.email)
        self.assertTrue(auth.has_access(user, CourseInstructorRole(self.course.id)))
        self.assertFalse(CourseStaffRole(self.course.id).has_user(user))

    def test_detail_post_instructor(self):
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "instructor"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assertFalse(CourseStaffRole(self.course.id).has_user(ext_user))
        self.assert_enrolled()

    def test_detail_post_missing_role(self):
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"toys": "fun"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        self.assert_not_enrolled()

    def test_detail_post_no_json(self):
        resp = self.client.post(
            self.detail_url,
            data={"role": "staff"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_enrolled()

    def test_detail_delete_staff(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.ext_user)

        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertFalse(auth.has_access(ext_user, CourseStaffRole(self.course.id)))

    def test_detail_delete_instructor(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.ext_user, self.user)

        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))

    def test_delete_last_instructor(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.ext_user)

        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))

    def test_post_last_instructor(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.ext_user)

        resp = self.client.post(
            self.detail_url,
            data={"role": "staff"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))

    def test_permission_denied_self(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user)
        self.user.is_staff = False
        self.user.save()

        self_url = self.course_team_url(email=self.user.email)

        resp = self.client.post(
            self_url,
            data={"role": "instructor"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)

    def test_permission_denied_other(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user)
        self.user.is_staff = False
        self.user.save()

        resp = self.client.post(
            self.detail_url,
            data={"role": "instructor"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)

    def test_staff_can_delete_self(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user)
        self.user.is_staff = False
        self.user.save()

        self_url = self.course_team_url(email=self.user.email)

        resp = self.client.delete(self_url)
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        user = User.objects.get(email=self.user.email)
        self.assertFalse(auth.has_access(user, CourseStaffRole(self.course.id)))

    def test_staff_cannot_delete_other(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user, self.ext_user)
        self.user.is_staff = False
        self.user.save()

        resp = self.client.delete(self.detail_url)
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))

    def test_user_not_initially_enrolled(self):
        # Verify that ext_user is not enrolled in the new course before being added as a staff member.
        self.assert_not_enrolled()

    def test_remove_staff_does_not_unenroll(self):
        # Add user with staff permissions.
        self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assert_enrolled()
        # Remove user from staff on course. Will not un-enroll them from the course.
        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        self.assert_enrolled()

    def test_staff_to_instructor_still_enrolled(self):
        # Add user with staff permission.
        self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assert_enrolled()
        # Now add with instructor permission. Verify still enrolled.
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "instructor"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        self.assert_enrolled()

    def assert_not_enrolled(self):
        """ Asserts that self.ext_user is not enrolled in self.course. """
        self.assertFalse(
            CourseEnrollment.is_enrolled(self.ext_user, self.course.id),
            'Did not expect ext_user to be enrolled in course'
        )

    def assert_enrolled(self):
        """ Asserts that self.ext_user is enrolled in self.course. """
        self.assertTrue(
            CourseEnrollment.is_enrolled(self.ext_user, self.course.id),
            'User ext_user should have been enrolled in the course'
        )

########NEW FILE########
__FILENAME__ = utils
"""
Utilities for view tests.
"""

import json

from contentstore.tests.utils import CourseTestCase
from contentstore.views.helpers import xblock_studio_url
from xmodule.modulestore.tests.factories import ItemFactory


class StudioPageTestCase(CourseTestCase):
    """
    Base class for all tests of Studio pages.
    """

    def setUp(self):
        super(StudioPageTestCase, self).setUp()
        self.chapter = ItemFactory.create(parent_location=self.course.location,
                                          category='chapter', display_name="Week 1")
        self.sequential = ItemFactory.create(parent_location=self.chapter.location,
                                             category='sequential', display_name="Lesson 1")

    def get_page_html(self, xblock):
        """
        Returns the HTML for the page representing the xblock.
        """
        url = xblock_studio_url(xblock)
        self.assertIsNotNone(url)
        resp = self.client.get_html(url)
        self.assertEqual(resp.status_code, 200)
        return resp.content

    def get_preview_html(self, xblock, view_name):
        """
        Returns the HTML for the xblock when shown within a unit or container page.
        """
        preview_url = '/xblock/{usage_key}/{view_name}'.format(usage_key=xblock.location, view_name=view_name)
        resp = self.client.get_json(preview_url)
        self.assertEqual(resp.status_code, 200)
        resp_content = json.loads(resp.content)
        return resp_content['html']

    def validate_preview_html(self, xblock, view_name, can_edit=True, can_reorder=True, can_add=True):
        """
        Verify that the specified xblock's preview has the expected HTML elements.
        """
        html = self.get_preview_html(xblock, view_name)
        self.validate_html_for_add_buttons(html, can_add=can_add)

        # Verify that there are no drag handles for public blocks
        drag_handle_html = '<span data-tooltip="Drag to reorder" class="drag-handle action"></span>'
        if can_reorder:
            self.assertIn(drag_handle_html, html)
        else:
            self.assertNotIn(drag_handle_html, html)

        # Verify that there are no action buttons for public blocks
        expected_button_html = [
            '<a href="#" class="edit-button action-button">',
            '<a href="#" data-tooltip="Delete" class="delete-button action-button">',
            '<a href="#" data-tooltip="Duplicate" class="duplicate-button action-button">'
        ]
        for button_html in expected_button_html:
            if can_edit:
                self.assertIn(button_html, html)
            else:
                self.assertNotIn(button_html, html)

    def validate_html_for_add_buttons(self, html, can_add=True):
        """
        Validate that the specified HTML has the appropriate add actions for the current publish state.
        """
        # Verify that there are no add buttons for public blocks
        add_button_html = '<div class="add-xblock-component new-component-item adding"></div>'
        if can_add:
            self.assertIn(add_button_html, html)
        else:
            self.assertNotIn(add_button_html, html)

########NEW FILE########
__FILENAME__ = transcripts_ajax
"""
Actions manager for transcripts ajax calls.
+++++++++++++++++++++++++++++++++++++++++++

Module do not support rollback (pressing "Cancel" button in Studio)
All user changes are saved immediately.
"""
import copy
import os
import logging
import json
import requests

from django.http import HttpResponse, Http404
from django.core.exceptions import PermissionDenied
from django.contrib.auth.decorators import login_required
from django.conf import settings
from django.utils.translation import ugettext as _

from opaque_keys import InvalidKeyError

from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import UsageKey
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.exceptions import ItemNotFoundError

from util.json_request import JsonResponse

from xmodule.video_module.transcripts_utils import (
    generate_subs_from_source,
    generate_srt_from_sjson, remove_subs_from_store,
    download_youtube_subs, get_transcripts_from_youtube,
    copy_or_rename_transcript,
    manage_video_subtitles_save,
    GetTranscriptsFromYouTubeException,
    TranscriptsRequestValidationException
)

from .access import has_course_access

__all__ = [
    'upload_transcripts',
    'download_transcripts',
    'check_transcripts',
    'choose_transcripts',
    'replace_transcripts',
    'rename_transcripts',
    'save_transcripts',
]

log = logging.getLogger(__name__)


def error_response(response, message, status_code=400):
    """
    Simplify similar actions: log message and return JsonResponse with message included in response.

    By default return 400 (Bad Request) Response.
    """
    log.debug(message)
    response['status'] = message
    return JsonResponse(response, status_code)


@login_required
def upload_transcripts(request):
    """
    Upload transcripts for current module.

    returns: response dict::

        status: 'Success' and HTTP 200 or 'Error' and HTTP 400.
        subs: Value of uploaded and saved html5 sub field in video item.
    """
    response = {
        'status': 'Unknown server error',
        'subs': '',
    }

    locator = request.POST.get('locator')
    if not locator:
        return error_response(response, 'POST data without "locator" form data.')

    try:
        item = _get_item(request, request.POST)
    except (InvalidKeyError, ItemNotFoundError):
        return error_response(response, "Can't find item by locator.")

    if 'transcript-file' not in request.FILES:
        return error_response(response, 'POST data without "file" form data.')

    video_list = request.POST.get('video_list')
    if not video_list:
        return error_response(response, 'POST data without video names.')

    try:
        video_list = json.loads(video_list)
    except ValueError:
        return error_response(response, 'Invalid video_list JSON.')

    source_subs_filedata = request.FILES['transcript-file'].read().decode('utf8')
    source_subs_filename = request.FILES['transcript-file'].name

    if '.' not in source_subs_filename:
        return error_response(response, "Undefined file extension.")

    basename = os.path.basename(source_subs_filename)
    source_subs_name = os.path.splitext(basename)[0]
    source_subs_ext = os.path.splitext(basename)[1][1:]

    if item.category != 'video':
        return error_response(response, 'Transcripts are supported only for "video" modules.')

    # Allow upload only if any video link is presented
    if video_list:
        sub_attr = source_subs_name
        try:
            # Generate and save for 1.0 speed, will create subs_sub_attr.srt.sjson subtitles file in storage.
            generate_subs_from_source({1: sub_attr}, source_subs_ext, source_subs_filedata, item)

            for video_dict in video_list:
                video_name = video_dict['video']
                # We are creating transcripts for every video source, if in future some of video sources would be deleted.
                # Updates item.sub with `video_name` on success.
                copy_or_rename_transcript(video_name, sub_attr, item, user=request.user)

            response['subs'] = item.sub
            response['status'] = 'Success'
        except Exception as ex:
            return error_response(response, ex.message)
    else:
        return error_response(response, 'Empty video sources.')

    return JsonResponse(response)


@login_required
def download_transcripts(request):
    """
    Passes to user requested transcripts file.

    Raises Http404 if unsuccessful.
    """
    locator = request.GET.get('locator')
    if not locator:
        log.debug('GET data without "locator" property.')
        raise Http404

    try:
        item = _get_item(request, request.GET)
    except (InvalidKeyError, ItemNotFoundError):
        log.debug("Can't find item by locator.")
        raise Http404

    subs_id = request.GET.get('subs_id')
    if not subs_id:
        log.debug('GET data without "subs_id" property.')
        raise Http404

    if item.category != 'video':
        log.debug('transcripts are supported only for video" modules.')
        raise Http404

    filename = 'subs_{0}.srt.sjson'.format(subs_id)
    content_location = StaticContent.compute_location(item.location.course_key, filename)
    try:
        sjson_transcripts = contentstore().find(content_location)
        log.debug("Downloading subs for %s id", subs_id)
        str_subs = generate_srt_from_sjson(json.loads(sjson_transcripts.data), speed=1.0)
        if not str_subs:
            log.debug('generate_srt_from_sjson produces no subtitles')
            raise Http404
        response = HttpResponse(str_subs, content_type='application/x-subrip')
        response['Content-Disposition'] = 'attachment; filename="{0}.srt"'.format(subs_id)
        return response
    except NotFoundError:
        log.debug("Can't find content in storage for %s subs", subs_id)
        raise Http404


@login_required
def check_transcripts(request):
    """
    Check state of transcripts availability.

    request.GET['data'] has key `videos`, which can contain any of the following::

        [
            {u'type': u'youtube', u'video': u'OEoXaMPEzfM', u'mode': u'youtube'},
            {u'type': u'html5',    u'video': u'video1',             u'mode': u'mp4'}
            {u'type': u'html5',    u'video': u'video2',             u'mode': u'webm'}
        ]
        `type` is youtube or html5
        `video` is html5 or youtube video_id
        `mode` is youtube, ,p4 or webm

    Returns transcripts_presence dict::

        html5_local: list of html5 ids, if subtitles exist locally for them;
        is_youtube_mode: bool, if we have youtube_id, and as youtube mode is of higher priority, reflect this with flag;
        youtube_local: bool, if youtube transcripts exist locally;
        youtube_server: bool, if youtube transcripts exist on server;
        youtube_diff: bool, if youtube transcripts exist on youtube server, and are different from local youtube ones;
        current_item_subs: string, value of item.sub field;
        status: string, 'Error' or 'Success';
        subs: string, new value of item.sub field, that should be set in module;
        command: string, action to front-end what to do and what to show to user.
    """
    transcripts_presence = {
        'html5_local': [],
        'html5_equal': False,
        'is_youtube_mode': False,
        'youtube_local': False,
        'youtube_server': False,
        'youtube_diff': True,
        'current_item_subs': None,
        'status': 'Error',
    }
    try:
        __, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(transcripts_presence, e.message)

    transcripts_presence['status'] = 'Success'

    filename = 'subs_{0}.srt.sjson'.format(item.sub)
    content_location = StaticContent.compute_location(item.location.course_key, filename)
    try:
        local_transcripts = contentstore().find(content_location).data
        transcripts_presence['current_item_subs'] = item.sub
    except NotFoundError:
        pass

    # Check for youtube transcripts presence
    youtube_id = videos.get('youtube', None)
    if youtube_id:
        transcripts_presence['is_youtube_mode'] = True

        # youtube local
        filename = 'subs_{0}.srt.sjson'.format(youtube_id)
        content_location = StaticContent.compute_location(item.location.course_key, filename)
        try:
            local_transcripts = contentstore().find(content_location).data
            transcripts_presence['youtube_local'] = True
        except NotFoundError:
            log.debug("Can't find transcripts in storage for youtube id: %s", youtube_id)

        # youtube server
        youtube_text_api = copy.deepcopy(settings.YOUTUBE['TEXT_API'])
        youtube_text_api['params']['v'] = youtube_id
        youtube_response = requests.get('http://' + youtube_text_api['url'], params=youtube_text_api['params'])

        if youtube_response.status_code == 200 and youtube_response.text:
            transcripts_presence['youtube_server'] = True
        #check youtube local and server transcripts for equality
        if transcripts_presence['youtube_server'] and transcripts_presence['youtube_local']:
            try:
                youtube_server_subs = get_transcripts_from_youtube(
                    youtube_id,
                    settings,
                    item.runtime.service(item, "i18n")
                )
                if json.loads(local_transcripts) == youtube_server_subs:  # check transcripts for equality
                    transcripts_presence['youtube_diff'] = False
            except GetTranscriptsFromYouTubeException:
                pass

    # Check for html5 local transcripts presence
    html5_subs = []
    for html5_id in videos['html5']:
        filename = 'subs_{0}.srt.sjson'.format(html5_id)
        content_location = StaticContent.compute_location(item.location.course_key, filename)
        try:
            html5_subs.append(contentstore().find(content_location).data)
            transcripts_presence['html5_local'].append(html5_id)
        except NotFoundError:
            log.debug("Can't find transcripts in storage for non-youtube video_id: %s", html5_id)
        if len(html5_subs) == 2:  # check html5 transcripts for equality
            transcripts_presence['html5_equal'] = json.loads(html5_subs[0]) == json.loads(html5_subs[1])

    command, subs_to_use = _transcripts_logic(transcripts_presence, videos)
    transcripts_presence.update({
        'command': command,
        'subs': subs_to_use,
    })
    return JsonResponse(transcripts_presence)


def _transcripts_logic(transcripts_presence, videos):
    """
    By `transcripts_presence` content, figure what show to user:

    returns: `command` and `subs`.

    `command`: string,  action to front-end what to do and what show to user.
    `subs`: string, new value of item.sub field, that should be set in module.

    `command` is one of::

        replace: replace local youtube subtitles with server one's
        found: subtitles are found
        import: import subtitles from youtube server
        choose: choose one from two html5 subtitles
        not found: subtitles are not found
    """
    command = None

    # new value of item.sub field, that should be set in module.
    subs = ''

    # youtube transcripts are of high priority than html5 by design
    if (
            transcripts_presence['youtube_diff'] and
            transcripts_presence['youtube_local'] and
            transcripts_presence['youtube_server']):  # youtube server and local exist
        command = 'replace'
        subs = videos['youtube']
    elif transcripts_presence['youtube_local']:  # only youtube local exist
        command = 'found'
        subs = videos['youtube']
    elif transcripts_presence['youtube_server']:  # only youtube server exist
        command = 'import'
    else:  # html5 part
        if transcripts_presence['html5_local']:  # can be 1 or 2 html5 videos
            if len(transcripts_presence['html5_local']) == 1 or transcripts_presence['html5_equal']:
                command = 'found'
                subs = transcripts_presence['html5_local'][0]
            else:
                command = 'choose'
                subs = transcripts_presence['html5_local'][0]
        else:  # html5 source have no subtitles
            # check if item sub has subtitles
            if transcripts_presence['current_item_subs'] and not transcripts_presence['is_youtube_mode']:
                log.debug("Command is use existing %s subs", transcripts_presence['current_item_subs'])
                command = 'use_existing'
            else:
                command = 'not_found'
    log.debug(
        "Resulted command: %s, current transcripts: %s, youtube mode: %s",
        command,
        transcripts_presence['current_item_subs'],
        transcripts_presence['is_youtube_mode']
    )
    return command, subs


@login_required
def choose_transcripts(request):
    """
    Replaces html5 subtitles, presented for both html5 sources, with chosen one.

    Code removes rejected html5 subtitles and updates sub attribute with chosen html5_id.

    It does nothing with youtube id's.

    Returns: status `Success` and resulted item.sub value or status `Error` and HTTP 400.
    """
    response = {
        'status': 'Error',
        'subs': '',
    }

    try:
        data, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(response, e.message)

    html5_id = data.get('html5_id')  # html5_id chosen by user

    # find rejected html5_id and remove appropriate subs from store
    html5_id_to_remove = [x for x in videos['html5'] if x != html5_id]
    if html5_id_to_remove:
        remove_subs_from_store(html5_id_to_remove, item)

    if item.sub != html5_id:  # update sub value
        item.sub = html5_id
        item.save_with_metadata(request.user)
    response = {'status': 'Success',  'subs': item.sub}
    return JsonResponse(response)


@login_required
def replace_transcripts(request):
    """
    Replaces all transcripts with youtube ones.

    Downloads subtitles from youtube and replaces all transcripts with downloaded ones.

    Returns: status `Success` and resulted item.sub value or status `Error` and HTTP 400.
    """
    response = {'status': 'Error', 'subs': ''}

    try:
        __, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(response, e.message)

    youtube_id = videos['youtube']
    if not youtube_id:
        return error_response(response, 'YouTube id {} is not presented in request data.'.format(youtube_id))

    try:
        download_youtube_subs({1.0: youtube_id}, item, settings)
    except GetTranscriptsFromYouTubeException as e:
        return error_response(response, e.message)

    item.sub = youtube_id
    item.save_with_metadata(request.user)
    response = {'status': 'Success',  'subs': item.sub}
    return JsonResponse(response)


def _validate_transcripts_data(request):
    """
    Validates, that request contains all proper data for transcripts processing.

    Returns tuple of 3 elements::

        data: dict, loaded json from request,
        videos: parsed `data` to useful format,
        item:  video item from storage

    Raises `TranscriptsRequestValidationException` if validation is unsuccessful
    or `PermissionDenied` if user has no access.
    """
    data = json.loads(request.GET.get('data', '{}'))
    if not data:
        raise TranscriptsRequestValidationException(_('Incoming video data is empty.'))

    try:
        item = _get_item(request, data)
    except (InvalidKeyError, ItemNotFoundError):
        raise TranscriptsRequestValidationException(_("Can't find item by locator."))

    if item.category != 'video':
        raise TranscriptsRequestValidationException(_('Transcripts are supported only for "video" modules.'))

    # parse data form request.GET.['data']['video'] to useful format
    videos = {'youtube': '', 'html5': {}}
    for video_data in data.get('videos'):
        if video_data['type'] == 'youtube':
            videos['youtube'] = video_data['video']
        else:  # do not add same html5 videos
            if videos['html5'].get('video') != video_data['video']:
                videos['html5'][video_data['video']] = video_data['mode']

    return data, videos, item


@login_required
def rename_transcripts(request):
    """
    Create copies of existing subtitles with new names of HTML5 sources.

    Old subtitles are not deleted now, because we do not have rollback functionality.

    If succeed, Item.sub will be chosen randomly from html5 video sources provided by front-end.
    """
    response = {'status': 'Error', 'subs': ''}

    try:
        __, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(response, e.message)

    old_name = item.sub

    for new_name in videos['html5'].keys():  # copy subtitles for every HTML5 source
        try:
            # updates item.sub with new_name if it is successful.
            copy_or_rename_transcript(new_name, old_name, item, user=request.user)
        except NotFoundError:
            # subtitles file `item.sub` is not presented in the system. Nothing to copy or rename.
            error_response(response, "Can't find transcripts in storage for {}".format(old_name))

    response['status'] = 'Success'
    response['subs'] = item.sub  # item.sub has been changed, it is not equal to old_name.
    log.debug("Updated item.sub to %s", item.sub)
    return JsonResponse(response)


@login_required
def save_transcripts(request):
    """
    Saves video module with updated values of fields.

    Returns: status `Success` or status `Error` and HTTP 400.
    """
    response = {'status': 'Error'}

    data = json.loads(request.GET.get('data', '{}'))
    if not data:
        return error_response(response, 'Incoming video data is empty.')

    try:
        item = _get_item(request, data)
    except (InvalidKeyError, ItemNotFoundError):
        return error_response(response, "Can't find item by locator.")

    metadata = data.get('metadata')
    if metadata is not None:
        new_sub = metadata.get('sub')

        for metadata_key, value in metadata.items():
            setattr(item, metadata_key, value)

        item.save_with_metadata(request.user)  # item becomes updated with new values

        if new_sub:
            manage_video_subtitles_save(item, request.user)
        else:
            # If `new_sub` is empty, it means that user explicitly does not want to use
            # transcripts for current video ids and we remove all transcripts from storage.
            current_subs = data.get('current_subs')
            if current_subs is not None:
                for sub in current_subs:
                    remove_subs_from_store(sub, item)

        response['status'] = 'Success'

    return JsonResponse(response)


def _get_item(request, data):
    """
    Obtains from 'data' the locator for an item.
    Next, gets that item from the modulestore (allowing any errors to raise up).
    Finally, verifies that the user has access to the item.

    Returns the item.
    """
    usage_key = UsageKey.from_string(data.get('locator'))

    # This is placed before has_course_access() to validate the location,
    # because has_course_access() raises  r if location is invalid.
    item = modulestore().get_item(usage_key)

    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    return item

########NEW FILE########
__FILENAME__ = user
from django.core.exceptions import PermissionDenied
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from django_future.csrf import ensure_csrf_cookie
from edxmako.shortcuts import render_to_response

from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from util.json_request import JsonResponse, expect_json
from student.roles import CourseInstructorRole, CourseStaffRole
from course_creators.views import user_requested_access

from .access import has_course_access

from student.models import CourseEnrollment
from django.http import HttpResponseNotFound
from student import auth


__all__ = ['request_course_creator', 'course_team_handler']


@require_POST
@login_required
def request_course_creator(request):
    """
    User has requested course creation access.
    """
    user_requested_access(request.user)
    return JsonResponse({"Status": "OK"})


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
def course_team_handler(request, course_key_string=None, email=None):
    """
    The restful handler for course team users.

    GET
        html: return html page for managing course team
        json: return json representation of a particular course team member (email is required).
    POST or PUT
        json: modify the permissions for a particular course team member (email is required, as well as role in the payload).
    DELETE:
        json: remove a particular course team member from the course team (email is required).
    """
    course_key = CourseKey.from_string(course_key_string) if course_key_string else None
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        return _course_team_user(request, course_key, email)
    elif request.method == 'GET':  # assume html
        return _manage_users(request, course_key)
    else:
        return HttpResponseNotFound()


def _manage_users(request, course_key):
    """
    This view will return all CMS users who are editors for the specified course
    """
    # check that logged in user has permissions to this item
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)
    instructors = CourseInstructorRole(course_key).users_with_role()
    # the page only lists staff and assumes they're a superset of instructors. Do a union to ensure.
    staff = set(CourseStaffRole(course_key).users_with_role()).union(instructors)

    return render_to_response('manage_users.html', {
        'context_course': course_module,
        'staff': staff,
        'instructors': instructors,
        'allow_actions': has_course_access(request.user, course_key, role=CourseInstructorRole),
    })


@expect_json
def _course_team_user(request, course_key, email):
    """
    Handle the add, remove, promote, demote requests ensuring the requester has authority
    """
    # check that logged in user has permissions to this item
    if has_course_access(request.user, course_key, role=CourseInstructorRole):
        # instructors have full permissions
        pass
    elif has_course_access(request.user, course_key, role=CourseStaffRole) and email == request.user.email:
        # staff can only affect themselves
        pass
    else:
        msg = {
            "error": _("Insufficient permissions")
        }
        return JsonResponse(msg, 400)


    try:
        user = User.objects.get(email=email)
    except Exception:
        msg = {
            "error": _("Could not find user by email address '{email}'.").format(email=email),
        }
        return JsonResponse(msg, 404)

    # role hierarchy: globalstaff > "instructor" > "staff" (in a course)
    if request.method == "GET":
        # just return info about the user
        msg = {
            "email": user.email,
            "active": user.is_active,
            "role": None,
        }
        # what's the highest role that this user has? (How should this report global staff?)
        for role in [CourseInstructorRole(course_key), CourseStaffRole(course_key)]:
            if role.has_user(user):
                msg["role"] = role.ROLE
                break
        return JsonResponse(msg)

    # can't modify an inactive user
    if not user.is_active:
        msg = {
            "error": _('User {email} has registered but has not yet activated his/her account.').format(email=email),
        }
        return JsonResponse(msg, 400)

    if request.method == "DELETE":
        try:
            try_remove_instructor(request, course_key, user)
        except CannotOrphanCourse as oops:
            return JsonResponse(oops.msg, 400)

        auth.remove_users(request.user, CourseStaffRole(course_key), user)
        return JsonResponse()

    # all other operations require the requesting user to specify a role
    role = request.json.get("role", request.POST.get("role"))
    if role is None:
        return JsonResponse({"error": _("`role` is required")}, 400)

    if role == "instructor":
        if not has_course_access(request.user, course_key, role=CourseInstructorRole):
            msg = {
                "error": _("Only instructors may create other instructors")
            }
            return JsonResponse(msg, 400)
        auth.add_users(request.user, CourseInstructorRole(course_key), user)
        # auto-enroll the course creator in the course so that "View Live" will work.
        CourseEnrollment.enroll(user, course_key)
    elif role == "staff":
        # add to staff regardless (can't do after removing from instructors as will no longer
        # be allowed)
        auth.add_users(request.user, CourseStaffRole(course_key), user)
        try:
            try_remove_instructor(request, course_key, user)
        except CannotOrphanCourse as oops:
            return JsonResponse(oops.msg, 400)

        # auto-enroll the course creator in the course so that "View Live" will work.
        CourseEnrollment.enroll(user, course_key)

    return JsonResponse()


class CannotOrphanCourse(Exception):
    """
    Exception raised if an attempt is made to remove all responsible instructors from course.
    """
    def __init__(self, msg):
        self.msg = msg
        Exception.__init__(self)


def try_remove_instructor(request, course_key, user):

    # remove all roles in this course from this user: but fail if the user
    # is the last instructor in the course team
    instructors = CourseInstructorRole(course_key)
    if instructors.has_user(user):
        if instructors.users_with_role().count() == 1:
            msg = {"error": _("You may not remove the last instructor from a course")}
            raise CannotOrphanCourse(msg)
        else:
            auth.remove_users(request.user, instructors, user)

########NEW FILE########
__FILENAME__ = xblock
"""
Views dedicated to rendering xblocks.
"""
from __future__ import absolute_import

import logging
import mimetypes

from xblock.core import XBlock

from django.conf import settings
from django.http import Http404, HttpResponse


log = logging.getLogger(__name__)


def xblock_resource(request, block_type, uri):  # pylint: disable=unused-argument
    """
    Return a package resource for the specified XBlock.
    """
    try:
        xblock_class = XBlock.load_class(block_type, settings.XBLOCK_SELECT_FUNCTION)
        content = xblock_class.open_local_resource(uri)
    except IOError:
        log.info('Failed to load xblock resource', exc_info=True)
        raise Http404
    except Exception:  # pylint: disable-msg=broad-except
        log.error('Failed to load xblock resource', exc_info=True)
        raise Http404

    mimetype, _ = mimetypes.guess_type(uri)
    return HttpResponse(content, mimetype=mimetype)

########NEW FILE########
__FILENAME__ = admin
"""
django admin page for the course creators table
"""

from course_creators.models import CourseCreator, update_creator_state, send_user_notification, send_admin_notification
from course_creators.views import update_course_creator_group

from ratelimitbackend import admin
from django.conf import settings
from django.dispatch import receiver
from edxmako.shortcuts import render_to_string
from django.core.mail import send_mail
from smtplib import SMTPException

import logging

log = logging.getLogger("studio.coursecreatoradmin")


def get_email(obj):
    """ Returns the email address for a user """
    return obj.user.email

get_email.short_description = 'email'


class CourseCreatorAdmin(admin.ModelAdmin):
    """
    Admin for the course creator table.
    """

    # Fields to display on the overview page.
    list_display = ['username', get_email, 'state', 'state_changed', 'note']
    readonly_fields = ['username', 'state_changed']
    # Controls the order on the edit form (without this, read-only fields appear at the end).
    fieldsets = (
        (None, {
            'fields': ['username', 'state', 'state_changed', 'note']
        }),
    )
    # Fields that filtering support
    list_filter = ['state', 'state_changed']
    # Fields that search supports.
    search_fields = ['user__username', 'user__email', 'state', 'note']
    # Turn off the action bar (we have no bulk actions)
    actions = None

    def username(self, inst):
        """
        Returns the username for a given user.

        Implemented to make sorting by username instead of by user object.
        """
        return inst.user.username

    username.admin_order_field = 'user__username'

    def has_add_permission(self, request):
        return False

    def has_delete_permission(self, request, obj=None):
        return False

    def has_change_permission(self, request, obj=None):
        return request.user.is_staff

    def save_model(self, request, obj, form, change):
        # Store who is making the request.
        obj.admin = request.user
        obj.save()


admin.site.register(CourseCreator, CourseCreatorAdmin)


@receiver(update_creator_state, sender=CourseCreator)
def update_creator_group_callback(sender, **kwargs):
    """
    Callback for when the model's creator status has changed.
    """
    user = kwargs['user']
    updated_state = kwargs['state']
    update_course_creator_group(kwargs['caller'], user, updated_state == CourseCreator.GRANTED)


@receiver(send_user_notification, sender=CourseCreator)
def send_user_notification_callback(sender, **kwargs):
    """
    Callback for notifying user about course creator status change.
    """
    user = kwargs['user']
    updated_state = kwargs['state']

    studio_request_email = settings.FEATURES.get('STUDIO_REQUEST_EMAIL', '')
    context = {'studio_request_email': studio_request_email}

    subject = render_to_string('emails/course_creator_subject.txt', context)
    subject = ''.join(subject.splitlines())
    if updated_state == CourseCreator.GRANTED:
        message_template = 'emails/course_creator_granted.txt'
    elif updated_state == CourseCreator.DENIED:
        message_template = 'emails/course_creator_denied.txt'
    else:
        # changed to unrequested or pending
        message_template = 'emails/course_creator_revoked.txt'
    message = render_to_string(message_template, context)

    try:
        user.email_user(subject, message, studio_request_email)
    except:
        log.warning("Unable to send course creator status e-mail to %s", user.email)


@receiver(send_admin_notification, sender=CourseCreator)
def send_admin_notification_callback(sender, **kwargs):
    """
    Callback for notifying admin of a user in the 'pending' state.
    """
    user = kwargs['user']

    studio_request_email = settings.FEATURES.get('STUDIO_REQUEST_EMAIL', '')
    context = {'user_name': user.username, 'user_email': user.email}

    subject = render_to_string('emails/course_creator_admin_subject.txt', context)
    subject = ''.join(subject.splitlines())
    message = render_to_string('emails/course_creator_admin_user_pending.txt', context)

    try:
        send_mail(
            subject,
            message,
            studio_request_email,
            [studio_request_email],
            fail_silently=False
        )
    except SMTPException:
        log.warning("Failure sending 'pending state' e-mail for %s to %s", user.email, studio_request_email)

########NEW FILE########
__FILENAME__ = 0001_initial
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'CourseCreator'
        db.create_table('course_creators_coursecreator', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], unique=True)),
            ('state_changed', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
            ('state', self.gf('django.db.models.fields.CharField')(default='unrequested', max_length=24)),
            ('note', self.gf('django.db.models.fields.CharField')(max_length=512, blank=True)),
        ))
        db.send_create_signal('course_creators', ['CourseCreator'])


    def backwards(self, orm):
        # Deleting model 'CourseCreator'
        db.delete_table('course_creators_coursecreator')


    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'course_creators.coursecreator': {
            'Meta': {'object_name': 'CourseCreator'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'note': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
            'state': ('django.db.models.fields.CharField', [], {'default': "'unrequested'", 'max_length': '24'}),
            'state_changed': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
        }
    }

    complete_apps = ['course_creators']
########NEW FILE########
__FILENAME__ = models
"""
Table for storing information about whether or not Studio users have course creation privileges.
"""
from django.db import models
from django.db.models.signals import post_init, post_save
from django.dispatch import receiver, Signal
from django.contrib.auth.models import User

from django.utils import timezone
from django.utils.translation import ugettext as _

# A signal that will be sent when users should be added or removed from the creator group
update_creator_state = Signal(providing_args=["caller", "user", "state"])

# A signal that will be sent when admin should be notified of a pending user request
send_admin_notification = Signal(providing_args=["user"])

# A signal that will be sent when user should be notified of change in course creator privileges
send_user_notification = Signal(providing_args=["user", "state"])


class CourseCreator(models.Model):
    """
    Creates the database table model.
    """
    UNREQUESTED = 'unrequested'
    PENDING = 'pending'
    GRANTED = 'granted'
    DENIED = 'denied'

    # Second value is the "human-readable" version.
    STATES = (
        (UNREQUESTED, _(u'unrequested')),
        (PENDING, _(u'pending')),
        (GRANTED, _(u'granted')),
        (DENIED, _(u'denied')),
    )

    user = models.ForeignKey(User, help_text=_("Studio user"), unique=True)
    state_changed = models.DateTimeField('state last updated', auto_now_add=True,
                                         help_text=_("The date when state was last updated"))
    state = models.CharField(max_length=24, blank=False, choices=STATES, default=UNREQUESTED,
                             help_text=_("Current course creator state"))
    note = models.CharField(max_length=512, blank=True, help_text=_("Optional notes about this user (for example, "
                                                                    "why course creation access was denied)"))

    def __unicode__(self):
        return u"{0} | {1} [{2}]".format(self.user, self.state, self.state_changed)


@receiver(post_init, sender=CourseCreator)
def post_init_callback(sender, **kwargs):
    """
    Extend to store previous state.
    """
    instance = kwargs['instance']
    instance.orig_state = instance.state


@receiver(post_save, sender=CourseCreator)
def post_save_callback(sender, **kwargs):
    """
    Extend to update state_changed time and fire event to update course creator group, if appropriate.
    """
    instance = kwargs['instance']
    # We only wish to modify the state_changed time if the state has been modified. We don't wish to
    # modify it for changes to the notes field.
    if instance.state != instance.orig_state:
        granted_state_change = instance.state == CourseCreator.GRANTED or instance.orig_state == CourseCreator.GRANTED
        # If either old or new state is 'granted', we must manipulate the course creator
        # group maintained by authz. That requires staff permissions (stored admin).
        if granted_state_change:
            assert hasattr(instance, 'admin'), 'Must have stored staff user to change course creator group'
            update_creator_state.send(
                sender=sender,
                caller=instance.admin,
                user=instance.user,
                state=instance.state
            )

        # If user has been denied access, granted access, or previously granted access has been
        # revoked, send a notification message to the user.
        if instance.state == CourseCreator.DENIED or granted_state_change:
            send_user_notification.send(
                sender=sender,
                user=instance.user,
                state=instance.state
            )

        # If the user has gone into the 'pending' state, send a notification to interested admin.
        if instance.state == CourseCreator.PENDING:
            send_admin_notification.send(
                sender=sender,
                user=instance.user
            )

        instance.state_changed = timezone.now()
        instance.orig_state = instance.state
        instance.save()

########NEW FILE########
__FILENAME__ = test_admin
"""
Tests course_creators.admin.py.
"""

from django.test import TestCase
from django.contrib.auth.models import User
from django.contrib.admin.sites import AdminSite
from django.http import HttpRequest
import mock

from course_creators.admin import CourseCreatorAdmin
from course_creators.models import CourseCreator
from django.core import mail
from student.roles import CourseCreatorRole
from student import auth


def mock_render_to_string(template_name, context):
    """Return a string that encodes template_name and context"""
    return str((template_name, context))


class CourseCreatorAdminTest(TestCase):
    """
    Tests for course creator admin.
    """

    def setUp(self):
        """ Test case setup """
        self.user = User.objects.create_user('test_user', 'test_user+courses@edx.org', 'foo')
        self.table_entry = CourseCreator(user=self.user)
        self.table_entry.save()

        self.admin = User.objects.create_user('Mark', 'admin+courses@edx.org', 'foo')
        self.admin.is_staff = True

        self.request = HttpRequest()
        self.request.user = self.admin

        self.creator_admin = CourseCreatorAdmin(self.table_entry, AdminSite())

        self.studio_request_email = 'mark@marky.mark'
        self.enable_creator_group_patch = {
            "ENABLE_CREATOR_GROUP": True,
            "STUDIO_REQUEST_EMAIL": self.studio_request_email
        }

    @mock.patch('course_creators.admin.render_to_string', mock.Mock(side_effect=mock_render_to_string, autospec=True))
    @mock.patch('django.contrib.auth.models.User.email_user')
    def test_change_status(self, email_user):
        """
        Tests that updates to state impact the creator group maintained in authz.py and that e-mails are sent.
        """

        def change_state_and_verify_email(state, is_creator):
            """ Changes user state, verifies creator status, and verifies e-mail is sent based on transition """
            self._change_state(state)
            self.assertEqual(is_creator, auth.has_access(self.user, CourseCreatorRole()))

            context = {'studio_request_email': self.studio_request_email}
            if state == CourseCreator.GRANTED:
                template = 'emails/course_creator_granted.txt'
            elif state == CourseCreator.DENIED:
                template = 'emails/course_creator_denied.txt'
            else:
                template = 'emails/course_creator_revoked.txt'
            email_user.assert_called_with(
                mock_render_to_string('emails/course_creator_subject.txt', context),
                mock_render_to_string(template, context),
                self.studio_request_email
            )

        with mock.patch.dict('django.conf.settings.FEATURES', self.enable_creator_group_patch):

            # User is initially unrequested.
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))

            change_state_and_verify_email(CourseCreator.GRANTED, True)

            change_state_and_verify_email(CourseCreator.DENIED, False)

            change_state_and_verify_email(CourseCreator.GRANTED, True)

            change_state_and_verify_email(CourseCreator.PENDING, False)

            change_state_and_verify_email(CourseCreator.GRANTED, True)

            change_state_and_verify_email(CourseCreator.UNREQUESTED, False)

            change_state_and_verify_email(CourseCreator.DENIED, False)

    @mock.patch('course_creators.admin.render_to_string', mock.Mock(side_effect=mock_render_to_string, autospec=True))
    def test_mail_admin_on_pending(self):
        """
        Tests that the admin account is notified when a user is in the 'pending' state.
        """

        def check_admin_message_state(state, expect_sent_to_admin, expect_sent_to_user):
            """ Changes user state and verifies e-mail sent to admin address only when pending. """
            mail.outbox = []
            self._change_state(state)

            # If a message is sent to the user about course creator status change, it will be the first
            # message sent. Admin message will follow.
            base_num_emails = 1 if expect_sent_to_user else 0
            if expect_sent_to_admin:
                context = {'user_name': "test_user", 'user_email': 'test_user+courses@edx.org'}
                self.assertEquals(base_num_emails + 1, len(mail.outbox), 'Expected admin message to be sent')
                sent_mail = mail.outbox[base_num_emails]
                self.assertEquals(
                    mock_render_to_string('emails/course_creator_admin_subject.txt', context),
                    sent_mail.subject
                )
                self.assertEquals(
                    mock_render_to_string('emails/course_creator_admin_user_pending.txt', context),
                    sent_mail.body
                )
                self.assertEquals(self.studio_request_email, sent_mail.from_email)
                self.assertEqual([self.studio_request_email], sent_mail.to)
            else:
                self.assertEquals(base_num_emails, len(mail.outbox))

        with mock.patch.dict('django.conf.settings.FEATURES', self.enable_creator_group_patch):
            # E-mail message should be sent to admin only when new state is PENDING, regardless of what
            # previous state was (unless previous state was already PENDING).
            # E-mail message sent to user only on transition into and out of GRANTED state.
            check_admin_message_state(CourseCreator.UNREQUESTED, expect_sent_to_admin=False, expect_sent_to_user=False)
            check_admin_message_state(CourseCreator.PENDING, expect_sent_to_admin=True, expect_sent_to_user=False)
            check_admin_message_state(CourseCreator.GRANTED, expect_sent_to_admin=False, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.DENIED, expect_sent_to_admin=False, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.GRANTED, expect_sent_to_admin=False, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.PENDING, expect_sent_to_admin=True, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.PENDING, expect_sent_to_admin=False, expect_sent_to_user=False)
            check_admin_message_state(CourseCreator.DENIED, expect_sent_to_admin=False, expect_sent_to_user=True)

    def _change_state(self, state):
        """ Helper method for changing state """
        self.table_entry.state = state
        self.creator_admin.save_model(self.request, self.table_entry, None, True)

    def test_add_permission(self):
        """
        Tests that staff cannot add entries
        """
        self.assertFalse(self.creator_admin.has_add_permission(self.request))

    def test_delete_permission(self):
        """
        Tests that staff cannot delete entries
        """
        self.assertFalse(self.creator_admin.has_delete_permission(self.request))

    def test_change_permission(self):
        """
        Tests that only staff can change entries
        """
        self.assertTrue(self.creator_admin.has_change_permission(self.request))

        self.request.user = self.user
        self.assertFalse(self.creator_admin.has_change_permission(self.request))

    def test_rate_limit_login(self):
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}):
            post_params = {'username': self.user.username, 'password': 'wrong_password'}
            # try logging in 30 times, the default limit in the number of failed
            # login attempts in one 5 minute period before the rate gets limited
            for _ in xrange(30):
                response = self.client.post('/admin/', post_params)
                self.assertEquals(response.status_code, 200)

            response = self.client.post('/admin/', post_params)
            # Since we are using the default rate limit behavior, we are
            # expecting this to return a 403 error to indicate that there have
            # been too many attempts
            self.assertEquals(response.status_code, 403)

########NEW FILE########
__FILENAME__ = test_views
"""
Tests course_creators.views.py.
"""

from django.test import TestCase
from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied

from course_creators.views import add_user_with_status_unrequested, add_user_with_status_granted
from course_creators.views import get_course_creator_status, update_course_creator_group, user_requested_access
import mock
from student.roles import CourseCreatorRole
from student import auth


class CourseCreatorView(TestCase):
    """
    Tests for modifying the course creator table.
    """

    def setUp(self):
        """ Test case setup """
        self.user = User.objects.create_user('test_user', 'test_user+courses@edx.org', 'foo')
        self.admin = User.objects.create_user('Mark', 'admin+courses@edx.org', 'foo')
        self.admin.is_staff = True

    def test_staff_permission_required(self):
        """
        Tests that any method changing the course creator authz group must be called with staff permissions.
        """
        with self.assertRaises(PermissionDenied):
            add_user_with_status_granted(self.user, self.user)

        with self.assertRaises(PermissionDenied):
            update_course_creator_group(self.user, self.user, True)

    def test_table_initially_empty(self):
        self.assertIsNone(get_course_creator_status(self.user))

    def test_add_unrequested(self):
        add_user_with_status_unrequested(self.user)
        self.assertEqual('unrequested', get_course_creator_status(self.user))

        # Calling add again will be a no-op (even if state is different).
        add_user_with_status_granted(self.admin, self.user)
        self.assertEqual('unrequested', get_course_creator_status(self.user))

    def test_add_granted(self):
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            # Calling add_user_with_status_granted impacts is_user_in_course_group_role.
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))

            add_user_with_status_granted(self.admin, self.user)
            self.assertEqual('granted', get_course_creator_status(self.user))

            # Calling add again will be a no-op (even if state is different).
            add_user_with_status_unrequested(self.user)
            self.assertEqual('granted', get_course_creator_status(self.user))

            self.assertTrue(auth.has_access(self.user, CourseCreatorRole()))

    def test_update_creator_group(self):
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))
            update_course_creator_group(self.admin, self.user, True)
            self.assertTrue(auth.has_access(self.user, CourseCreatorRole()))
            update_course_creator_group(self.admin, self.user, False)
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))

    def test_user_requested_access(self):
        add_user_with_status_unrequested(self.user)
        self.assertEqual('unrequested', get_course_creator_status(self.user))
        user_requested_access(self.user)
        self.assertEqual('pending', get_course_creator_status(self.user))

    def test_user_requested_already_granted(self):
        add_user_with_status_granted(self.admin, self.user)
        self.assertEqual('granted', get_course_creator_status(self.user))
        # Will not "downgrade" to pending because that would require removing the
        # user from the authz course creator group (and that can only be done by an admin).
        user_requested_access(self.user)
        self.assertEqual('granted', get_course_creator_status(self.user))

    def test_add_user_unrequested_staff(self):
        # Users marked as is_staff will not be added to the course creator table.
        add_user_with_status_unrequested(self.admin)
        self.assertIsNone(get_course_creator_status(self.admin))

    def test_add_user_granted_staff(self):
        # Users marked as is_staff will not be added to the course creator table.
        add_user_with_status_granted(self.admin, self.admin)
        self.assertIsNone(get_course_creator_status(self.admin))

########NEW FILE########
__FILENAME__ = views
"""
Methods for interacting programmatically with the user creator table.
"""
from course_creators.models import CourseCreator
from student.roles import CourseCreatorRole
from student import auth


def add_user_with_status_unrequested(user):
    """
    Adds a user to the course creator table with status 'unrequested'.

    If the user is already in the table, this method is a no-op
    (state will not be changed).

    If the user is marked as is_staff, this method is a no-op (user
    will not be added to table).
    """
    _add_user(user, CourseCreator.UNREQUESTED)


def add_user_with_status_granted(caller, user):
    """
    Adds a user to the course creator table with status 'granted'.

    If appropriate, this method also adds the user to the course creator group maintained by authz.py.
    Caller must have staff permissions.

    If the user is already in the table, this method is a no-op
    (state will not be changed).

    If the user is marked as is_staff, this method is a no-op (user
    will not be added to table, nor added to authz.py group).
    """
    if _add_user(user, CourseCreator.GRANTED):
        update_course_creator_group(caller, user, True)


def update_course_creator_group(caller, user, add):
    """
    Method for adding and removing users from the creator group.

    Caller must have staff permissions.
    """
    if add:
        auth.add_users(caller, CourseCreatorRole(), user)
    else:
        auth.remove_users(caller, CourseCreatorRole(), user)


def get_course_creator_status(user):
    """
    Returns the status for a particular user, or None if user is not in the table.

    Possible return values are:
        'unrequested' = user has not requested course creation rights
        'pending' = user has requested course creation rights
        'granted' = user has been granted course creation rights
        'denied' = user has been denied course creation rights
        None = user does not exist in the table
    """
    user = CourseCreator.objects.filter(user=user)
    if user.count() == 0:
        return None
    else:
        # User is defined to be unique, can assume a single entry.
        return user[0].state


def user_requested_access(user):
    """
    User has requested course creator access.

    This changes the user state to CourseCreator.PENDING, unless the user
    state is already CourseCreator.GRANTED, in which case this method is a no-op.
    """
    user = CourseCreator.objects.get(user=user)
    if user.state != CourseCreator.GRANTED:
        user.state = CourseCreator.PENDING
        user.save()


def _add_user(user, state):
    """
    Adds a user to the course creator table with the specified state.

    Returns True if user was added to table, else False.

    If the user is already in the table, this method is a no-op
    (state will not be changed, method will return False).

    If the user is marked as is_staff, this method is a no-op (False will be returned).
    """
    if not user.is_staff and CourseCreator.objects.filter(user=user).count() == 0:
        entry = CourseCreator(user=user, state=state)
        entry.save()
        return True

    return False

########NEW FILE########
__FILENAME__ = course_details
import re
import logging
import datetime
import json
from json.encoder import JSONEncoder

from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError
from contentstore.utils import get_modulestore, course_image_url
from models.settings import course_grading
from xmodule.fields import Date
from xmodule.modulestore.django import modulestore

class CourseDetails(object):
    def __init__(self, org, course_id, run):
        # still need these for now b/c the client's screen shows these 3 fields
        self.org = org
        self.course_id = course_id
        self.run = run
        self.start_date = None  # 'start'
        self.end_date = None  # 'end'
        self.enrollment_start = None
        self.enrollment_end = None
        self.syllabus = None  # a pdf file asset
        self.short_description = ""
        self.overview = ""  # html to render as the overview
        self.intro_video = None  # a video pointer
        self.effort = None  # int hours/week
        self.course_image_name = ""
        self.course_image_asset_path = ""  # URL of the course image

    @classmethod
    def fetch(cls, course_key):
        """
        Fetch the course details for the given course from persistence and return a CourseDetails model.
        """
        descriptor = modulestore('direct').get_course(course_key)
        course_details = cls(course_key.org, course_key.course, course_key.run)

        course_details.start_date = descriptor.start
        course_details.end_date = descriptor.end
        course_details.enrollment_start = descriptor.enrollment_start
        course_details.enrollment_end = descriptor.enrollment_end
        course_details.course_image_name = descriptor.course_image
        course_details.course_image_asset_path = course_image_url(descriptor)

        temploc = course_key.make_usage_key('about', 'syllabus')
        try:
            course_details.syllabus = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'short_description')
        try:
            course_details.short_description = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'overview')
        try:
            course_details.overview = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'effort')
        try:
            course_details.effort = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'video')
        try:
            raw_video = get_modulestore(temploc).get_item(temploc).data
            course_details.intro_video = CourseDetails.parse_video_tag(raw_video)
        except ItemNotFoundError:
            pass

        return course_details

    @classmethod
    def update_about_item(cls, course_key, about_key, data, course, user):
        """
        Update the about item with the new data blob. If data is None, then
        delete the about item.
        """
        temploc = course_key.make_usage_key('about', about_key)
        store = get_modulestore(temploc)
        if data is None:
            store.delete_item(temploc)
        else:
            try:
                about_item = store.get_item(temploc)
            except ItemNotFoundError:
                about_item = store.create_xmodule(temploc, system=course.runtime)
            about_item.data = data
            store.update_item(about_item, user.id)

    @classmethod
    def update_from_json(cls, course_key, jsondict, user):
        """
        Decode the json into CourseDetails and save any changed attrs to the db
        """
        module_store = modulestore('direct')
        descriptor = module_store.get_course(course_key)

        dirty = False

        # In the descriptor's setter, the date is converted to JSON using Date's to_json method.
        # Calling to_json on something that is already JSON doesn't work. Since reaching directly
        # into the model is nasty, convert the JSON Date to a Python date, which is what the
        # setter expects as input.
        date = Date()

        if 'start_date' in jsondict:
            converted = date.from_json(jsondict['start_date'])
        else:
            converted = None
        if converted != descriptor.start:
            dirty = True
            descriptor.start = converted

        if 'end_date' in jsondict:
            converted = date.from_json(jsondict['end_date'])
        else:
            converted = None

        if converted != descriptor.end:
            dirty = True
            descriptor.end = converted

        if 'enrollment_start' in jsondict:
            converted = date.from_json(jsondict['enrollment_start'])
        else:
            converted = None

        if converted != descriptor.enrollment_start:
            dirty = True
            descriptor.enrollment_start = converted

        if 'enrollment_end' in jsondict:
            converted = date.from_json(jsondict['enrollment_end'])
        else:
            converted = None

        if converted != descriptor.enrollment_end:
            dirty = True
            descriptor.enrollment_end = converted

        if 'course_image_name' in jsondict and jsondict['course_image_name'] != descriptor.course_image:
            descriptor.course_image = jsondict['course_image_name']
            dirty = True

        if dirty:
            module_store.update_item(descriptor, user.id)

        # NOTE: below auto writes to the db w/o verifying that any of the fields actually changed
        # to make faster, could compare against db or could have client send over a list of which fields changed.
        for about_type in ['syllabus', 'overview', 'effort', 'short_description']:
            cls.update_about_item(course_key, about_type, jsondict[about_type], descriptor, user)

        recomposed_video_tag = CourseDetails.recompose_video_tag(jsondict['intro_video'])
        cls.update_about_item(course_key, 'video', recomposed_video_tag, descriptor, user)

        # Could just return jsondict w/o doing any db reads, but I put the reads in as a means to confirm
        # it persisted correctly
        return CourseDetails.fetch(course_key)

    @staticmethod
    def parse_video_tag(raw_video):
        """
        Because the client really only wants the author to specify the youtube key, that's all we send to and get from the client.
        The problem is that the db stores the html markup as well (which, of course, makes any sitewide changes to how we do videos
        next to impossible.)
        """
        if not raw_video:
            return None

        keystring_matcher = re.search(r'(?<=embed/)[a-zA-Z0-9_-]+', raw_video)
        if keystring_matcher is None:
            keystring_matcher = re.search(r'<?=\d+:[a-zA-Z0-9_-]+', raw_video)

        if keystring_matcher:
            return keystring_matcher.group(0)
        else:
            logging.warn("ignoring the content because it doesn't not conform to expected pattern: " + raw_video)
            return None

    @staticmethod
    def recompose_video_tag(video_key):
        # TODO should this use a mako template? Of course, my hope is that this is a short-term workaround for the db not storing
        # the right thing
        result = None
        if video_key:
            result = '<iframe width="560" height="315" src="//www.youtube.com/embed/' + \
                video_key + '?rel=0" frameborder="0" allowfullscreen=""></iframe>'
        return result


# TODO move to a more general util?
class CourseSettingsEncoder(json.JSONEncoder):
    """
    Serialize CourseDetails, CourseGradingModel, datetime, and old Locations
    """
    def default(self, obj):
        if isinstance(obj, (CourseDetails, course_grading.CourseGradingModel)):
            return obj.__dict__
        elif isinstance(obj, Location):
            return obj.dict()
        elif isinstance(obj, datetime.datetime):
            return Date().to_json(obj)
        else:
            return JSONEncoder.default(self, obj)

########NEW FILE########
__FILENAME__ = course_grading
from datetime import timedelta
from xmodule.modulestore.django import modulestore
from xblock.fields import Scope


class CourseGradingModel(object):
    """
    Basically a DAO and Model combo for CRUD operations pertaining to grading policy.
    """
    # Within this class, allow access to protected members of client classes.
    # This comes up when accessing kvs data and caches during kvs saves and modulestore writes.
    def __init__(self, course_descriptor):
        self.graders = [
            CourseGradingModel.jsonize_grader(i, grader) for i, grader in enumerate(course_descriptor.raw_grader)
        ]  # weights transformed to ints [0..100]
        self.grade_cutoffs = course_descriptor.grade_cutoffs
        self.grace_period = CourseGradingModel.convert_set_grace_period(course_descriptor)

    @classmethod
    def fetch(cls, course_key):
        """
        Fetch the course grading policy for the given course from persistence and return a CourseGradingModel.
        """
        descriptor = modulestore('direct').get_course(course_key)
        model = cls(descriptor)
        return model

    @staticmethod
    def fetch_grader(course_key, index):
        """
        Fetch the course's nth grader
        Returns an empty dict if there's no such grader.
        """
        descriptor = modulestore('direct').get_course(course_key)
        index = int(index)
        if len(descriptor.raw_grader) > index:
            return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])

        # return empty model
        else:
            return {"id": index,
                    "type": "",
                    "min_count": 0,
                    "drop_count": 0,
                    "short_label": None,
                    "weight": 0
                    }

    @staticmethod
    def update_from_json(course_key, jsondict, user):
        """
        Decode the json into CourseGradingModel and save any changes. Returns the modified model.
        Probably not the usual path for updates as it's too coarse grained.
        """
        descriptor = modulestore('direct').get_course(course_key)

        graders_parsed = [CourseGradingModel.parse_grader(jsonele) for jsonele in jsondict['graders']]

        descriptor.raw_grader = graders_parsed
        descriptor.grade_cutoffs = jsondict['grade_cutoffs']

        modulestore('direct').update_item(descriptor, user.id)

        CourseGradingModel.update_grace_period_from_json(course_key, jsondict['grace_period'], user)

        return CourseGradingModel.fetch(course_key)

    @staticmethod
    def update_grader_from_json(course_key, grader, user):
        """
        Create or update the grader of the given type (string key) for the given course. Returns the modified
        grader which is a full model on the client but not on the server (just a dict)
        """
        descriptor = modulestore('direct').get_course(course_key)

        # parse removes the id; so, grab it before parse
        index = int(grader.get('id', len(descriptor.raw_grader)))
        grader = CourseGradingModel.parse_grader(grader)

        if index < len(descriptor.raw_grader):
            descriptor.raw_grader[index] = grader
        else:
            descriptor.raw_grader.append(grader)

        modulestore('direct').update_item(descriptor, user.id)

        return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])

    @staticmethod
    def update_cutoffs_from_json(course_key, cutoffs, user):
        """
        Create or update the grade cutoffs for the given course. Returns sent in cutoffs (ie., no extra
        db fetch).
        """
        descriptor = modulestore('direct').get_course(course_key)
        descriptor.grade_cutoffs = cutoffs

        modulestore('direct').update_item(descriptor, user.id)

        return cutoffs

    @staticmethod
    def update_grace_period_from_json(course_key, graceperiodjson, user):
        """
        Update the course's default grace period. Incoming dict is {hours: h, minutes: m} possibly as a
        grace_period entry in an enclosing dict. It is also safe to call this method with a value of
        None for graceperiodjson.
        """
        descriptor = modulestore('direct').get_course(course_key)

        # Before a graceperiod has ever been created, it will be None (once it has been
        # created, it cannot be set back to None).
        if graceperiodjson is not None:
            if 'grace_period' in graceperiodjson:
                graceperiodjson = graceperiodjson['grace_period']

            grace_timedelta = timedelta(**graceperiodjson)
            descriptor.graceperiod = grace_timedelta

            modulestore('direct').update_item(descriptor, user.id)

    @staticmethod
    def delete_grader(course_key, index, user):
        """
        Delete the grader of the given type from the given course.
        """
        descriptor = modulestore('direct').get_course(course_key)

        index = int(index)
        if index < len(descriptor.raw_grader):
            del descriptor.raw_grader[index]
            # force propagation to definition
            descriptor.raw_grader = descriptor.raw_grader

        modulestore('direct').update_item(descriptor, user.id)

    @staticmethod
    def delete_grace_period(course_key, user):
        """
        Delete the course's grace period.
        """
        descriptor = modulestore('direct').get_course(course_key)

        del descriptor.graceperiod

        modulestore('direct').update_item(descriptor, user.id)

    @staticmethod
    def get_section_grader_type(location):
        descriptor = modulestore('direct').get_item(location)
        return {
            "graderType": descriptor.format if descriptor.format is not None else 'notgraded',
            "location": unicode(location),
        }

    @staticmethod
    def update_section_grader_type(descriptor, grader_type, user):
        if grader_type is not None and grader_type != u'notgraded':
            descriptor.format = grader_type
            descriptor.graded = True
        else:
            del descriptor.format
            del descriptor.graded

        modulestore('direct').update_item(descriptor, user.id)
        return {'graderType': grader_type}

    @staticmethod
    def convert_set_grace_period(descriptor):
        # 5 hours 59 minutes 59 seconds => converted to iso format
        rawgrace = descriptor.graceperiod
        if rawgrace:
            hours_from_days = rawgrace.days * 24
            seconds = rawgrace.seconds
            hours_from_seconds = int(seconds / 3600)
            hours = hours_from_days + hours_from_seconds
            seconds -= hours_from_seconds * 3600
            minutes = int(seconds / 60)
            seconds -= minutes * 60

            graceperiod = {'hours': 0, 'minutes': 0, 'seconds': 0}
            if hours > 0:
                graceperiod['hours'] = hours

            if minutes > 0:
                graceperiod['minutes'] = minutes

            if seconds > 0:
                graceperiod['seconds'] = seconds

            return graceperiod
        else:
            return None

    @staticmethod
    def parse_grader(json_grader):
        # manual to clear out kruft
        result = {"type": json_grader["type"],
                  "min_count": int(json_grader.get('min_count', 0)),
                  "drop_count": int(json_grader.get('drop_count', 0)),
                  "short_label": json_grader.get('short_label', None),
                  "weight": float(json_grader.get('weight', 0)) / 100.0
                  }

        return result

    @staticmethod
    def jsonize_grader(i, grader):
        grader['id'] = i
        if grader['weight']:
            grader['weight'] *= 100
        if not 'short_label' in grader:
            grader['short_label'] = ""

        return grader

########NEW FILE########
__FILENAME__ = course_metadata
from xblock.fields import Scope

from contentstore.utils import get_modulestore
from cms.lib.xblock.mixin import CmsBlockMixin


class CourseMetadata(object):
    '''
    For CRUD operations on metadata fields which do not have specific editors
    on the other pages including any user generated ones.
    The objects have no predefined attrs but instead are obj encodings of the
    editable metadata.
    '''
    FILTERED_LIST = ['xml_attributes',
                     'start',
                     'end',
                     'enrollment_start',
                     'enrollment_end',
                     'tabs',
                     'graceperiod',
                     'checklists',
                     'show_timezone',
                     'format',
                     'graded',
    ]

    @classmethod
    def fetch(cls, descriptor):
        """
        Fetch the key:value editable course details for the given course from
        persistence and return a CourseMetadata model.
        """
        result = {}

        for field in descriptor.fields.values():
            if field.name in CmsBlockMixin.fields:
                continue

            if field.scope != Scope.settings:
                continue

            if field.name in cls.FILTERED_LIST:
                continue

            result[field.name] = field.read_json(descriptor)

        return result

    @classmethod
    def update_from_json(cls, descriptor, jsondict, filter_tabs=True, user=None):
        """
        Decode the json into CourseMetadata and save any changed attrs to the db.

        Ensures none of the fields are in the blacklist.
        """
        dirty = False

        # Copy the filtered list to avoid permanently changing the class attribute.
        filtered_list = list(cls.FILTERED_LIST)
        # Don't filter on the tab attribute if filter_tabs is False.
        if not filter_tabs:
            filtered_list.remove("tabs")

        for key, val in jsondict.iteritems():
            # should it be an error if one of the filtered list items is in the payload?
            if key in filtered_list:
                continue

            if key == "unsetKeys":
                dirty = True
                for unset in val:
                    descriptor.fields[unset].delete_from(descriptor)

            if hasattr(descriptor, key) and getattr(descriptor, key) != val:
                dirty = True
                value = descriptor.fields[key].from_json(val)
                setattr(descriptor, key, value)

        if dirty:
            get_modulestore(descriptor.location).update_item(descriptor, user.id if user else None)

        return cls.fetch(descriptor)

########NEW FILE########
__FILENAME__ = acceptance
"""
This config file extends the test environment configuration
so that we can run the lettuce acceptance tests.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .test import *
from lms.envs.sauce import *

# You need to start the server in debug mode,
# otherwise the browser will not render the pages correctly
DEBUG = True

# Output Django logs to a file
import logging
logging.basicConfig(filename=TEST_ROOT / "log" / "cms_acceptance.log", level=logging.ERROR)

import os


def seed():
    return os.getppid()

# Suppress error message "Cannot determine primary key of logged in user"
# from track.middleware that gets triggered when using an auto_auth workflow
# This is an ERROR level warning so we need to set the threshold at CRITICAL
logging.getLogger('track.middleware').setLevel(logging.CRITICAL)

# Suppress warning message "Cannot find corresponding link for name: <foo>"
# from edxmako.shortcuts. We have no appropriate pages in the platform to
# use, so these are not set up for TOS and PRIVACY
logging.getLogger('edxmako.shortcuts').setLevel(logging.ERROR)

DOC_STORE_CONFIG = {
    'host': 'localhost',
    'db': 'acceptance_xmodule',
    'collection': 'acceptance_modulestore_%s' % seed(),
}

MODULESTORE_OPTIONS = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'fs_root': TEST_ROOT / "data",
    'render_template': 'edxmako.shortcuts.render_to_string',
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'draft': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    }
}

CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'DOC_STORE_CONFIG': {
        'host': 'localhost',
        'db': 'acceptance_xcontent_%s' % seed(),
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

# Set this up so that rake lms[acceptance] and running the
# harvest command both use the same (test) database
# which they can flush without messing up your dev db
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': TEST_ROOT / "db" / "test_edx.db",
        'TEST_NAME': TEST_ROOT / "db" / "test_edx.db"
    }
}

# Enable asset pipeline
# Our fork of django-pipeline uses `PIPELINE` instead of `PIPELINE_ENABLED`
# PipelineFinder is explained here: http://django-pipeline.readthedocs.org/en/1.1.24/storages.html
PIPELINE = True
STATICFILES_FINDERS += ('pipeline.finders.PipelineFinder', )

# Use the auto_auth workflow for creating users and logging them in
FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True

# Forums are disabled in test.py to speed up unit tests, but we do not have
# per-test control for lettuce acceptance tests.
# If you are writing an acceptance test that needs the discussion service enabled,
# do not write it in lettuce, but instead write it using bok-choy.
# DO NOT CHANGE THIS SETTING HERE.
FEATURES['ENABLE_DISCUSSION_SERVICE'] = False

# HACK
# Setting this flag to false causes imports to not load correctly in the lettuce python files
# We do not yet understand why this occurs. Setting this to true is a stopgap measure
USE_I18N = True

# Include the lettuce app for acceptance testing, including the 'harvest' django-admin command
INSTALLED_APPS += ('lettuce.django',)
LETTUCE_APPS = ('contentstore',)
LETTUCE_BROWSER = os.environ.get('LETTUCE_BROWSER', 'chrome')

# Where to run: local, saucelabs, or grid
LETTUCE_SELENIUM_CLIENT = os.environ.get('LETTUCE_SELENIUM_CLIENT', 'local')

SELENIUM_GRID = {
    'URL': 'http://127.0.0.1:4444/wd/hub',
    'BROWSER': LETTUCE_BROWSER,
}

#####################################################################
# Lastly, see if the developer has any local overrides.
try:
    from .private import *  # pylint: disable=F0401
except ImportError:
    pass

# Point the URL used to test YouTube availability to our stub YouTube server
YOUTUBE['API'] = "127.0.0.1:{0}/get_youtube_api/".format(YOUTUBE_PORT)
YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)

########NEW FILE########
__FILENAME__ = aws
"""
This is the default template for our main set of AWS servers.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

import json

from .common import *

from logsettings import get_logger_config
import os

from path import path
from dealer.git import git

# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
# configuration files are read during startup.
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)

# CONFIG_ROOT specifies the directory where the JSON configuration
# files are expected to be found. If not specified, use the project
# directory.
CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT))

# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
# based on the service variant. If no variant is use, don't use a
# prefix.
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""


############### ALWAYS THE SAME ################################

DEBUG = False
TEMPLATE_DEBUG = False

EMAIL_BACKEND = 'django_ses.SESBackend'
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'

###################################### CELERY  ################################

# Don't use a connection pool, since connections are dropped by ELB.
BROKER_POOL_LIMIT = 0
BROKER_CONNECTION_TIMEOUT = 1

# For the Result Store, use the django cache named 'celery'
CELERY_RESULT_BACKEND = 'cache'
CELERY_CACHE_BACKEND = 'celery'

# When the broker is behind an ELB, use a heartbeat to refresh the
# connection and to detect if it has been dropped.
BROKER_HEARTBEAT = 10.0
BROKER_HEARTBEAT_CHECKRATE = 2

# Each worker should only fetch one message at a time
CELERYD_PREFETCH_MULTIPLIER = 1

# Skip djcelery migrations, since we don't use the database as the broker
SOUTH_MIGRATION_MODULES = {
    'djcelery': 'ignore',
}

# Rename the exchange and queues for each variant

QUEUE_VARIANT = CONFIG_PREFIX.lower()

CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT)

HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT)
DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT)
LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT)

CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE

CELERY_QUEUES = {
    HIGH_PRIORITY_QUEUE: {},
    LOW_PRIORITY_QUEUE: {},
    DEFAULT_PRIORITY_QUEUE: {}
}

############# NON-SECURE ENV CONFIG ##############################
# Things like server locations, ports, etc.
with open(CONFIG_ROOT / CONFIG_PREFIX + "env.json") as env_file:
    ENV_TOKENS = json.load(env_file)

# STATIC_URL_BASE specifies the base url to use for static files
STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None)
if STATIC_URL_BASE:
    # collectstatic will fail if STATIC_URL is a unicode string
    STATIC_URL = STATIC_URL_BASE.encode('ascii')
    if not STATIC_URL.endswith("/"):
        STATIC_URL += "/"
    STATIC_URL += git.revision + "/"

# GITHUB_REPO_ROOT is the base directory
# for course data
GITHUB_REPO_ROOT = ENV_TOKENS.get('GITHUB_REPO_ROOT', GITHUB_REPO_ROOT)

# STATIC_ROOT specifies the directory where static files are
# collected

STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None)
if STATIC_ROOT_BASE:
    STATIC_ROOT = path(STATIC_ROOT_BASE) / git.revision

EMAIL_BACKEND = ENV_TOKENS.get('EMAIL_BACKEND', EMAIL_BACKEND)
EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', None)

EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', EMAIL_HOST)
EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', EMAIL_PORT)
EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', EMAIL_USE_TLS)

LMS_BASE = ENV_TOKENS.get('LMS_BASE')
# Note that FEATURES['PREVIEW_LMS_BASE'] gets read in from the environment file.

SITE_NAME = ENV_TOKENS['SITE_NAME']

LOG_DIR = ENV_TOKENS['LOG_DIR']

CACHES = ENV_TOKENS['CACHES']
# Cache used for location mapping -- called many times with the same key/value
# in a given request.
if 'loc_cache' not in CACHES:
    CACHES['loc_cache'] = {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_location_mem_cache',
    }

SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
SESSION_ENGINE = ENV_TOKENS.get('SESSION_ENGINE', SESSION_ENGINE)

# allow for environments to specify what cookie name our login subsystem should use
# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can
# happen with some browsers (e.g. Firefox)
if ENV_TOKENS.get('SESSION_COOKIE_NAME', None):
    # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str()
    SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME'))

#Email overrides
DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL)
DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL)
ADMINS = ENV_TOKENS.get('ADMINS', ADMINS)
SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL)
MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS)
TECH_SUPPORT_EMAIL = ENV_TOKENS.get('TECH_SUPPORT_EMAIL', TECH_SUPPORT_EMAIL)

COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", [])

# Theme overrides
THEME_NAME = ENV_TOKENS.get('THEME_NAME', None)

#Timezone overrides
TIME_ZONE = ENV_TOKENS.get('TIME_ZONE', TIME_ZONE)

# Push to LMS overrides
GIT_REPO_EXPORT_DIR = ENV_TOKENS.get('GIT_REPO_EXPORT_DIR', '/edx/var/edxapp/export_course_repos')

# Translation overrides
LANGUAGES = ENV_TOKENS.get('LANGUAGES', LANGUAGES)
LANGUAGE_CODE = ENV_TOKENS.get('LANGUAGE_CODE', LANGUAGE_CODE)
USE_I18N = ENV_TOKENS.get('USE_I18N', USE_I18N)

ENV_FEATURES = ENV_TOKENS.get('FEATURES', ENV_TOKENS.get('MITX_FEATURES', {}))
for feature, value in ENV_FEATURES.items():
    FEATURES[feature] = value

WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED', WIKI_ENABLED)

LOGGING = get_logger_config(LOG_DIR,
                            logging_env=ENV_TOKENS['LOGGING_ENV'],
                            syslog_addr=(ENV_TOKENS['SYSLOG_SERVER'], 514),
                            debug=False,
                            service_variant=SERVICE_VARIANT)

#theming start:
PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME', 'edX')

# Event Tracking
if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
    TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")

# Django CAS external authentication settings
CAS_EXTRA_LOGIN_PARAMS = ENV_TOKENS.get("CAS_EXTRA_LOGIN_PARAMS", None)
if FEATURES.get('AUTH_USE_CAS'):
    CAS_SERVER_URL = ENV_TOKENS.get("CAS_SERVER_URL", None)
    AUTHENTICATION_BACKENDS = (
        'django.contrib.auth.backends.ModelBackend',
        'django_cas.backends.CASBackend',
    )
    INSTALLED_APPS += ('django_cas',)
    MIDDLEWARE_CLASSES += ('django_cas.middleware.CASMiddleware',)
    CAS_ATTRIBUTE_CALLBACK = ENV_TOKENS.get('CAS_ATTRIBUTE_CALLBACK', None)
    if CAS_ATTRIBUTE_CALLBACK:
        import importlib
        CAS_USER_DETAILS_RESOLVER = getattr(
            importlib.import_module(CAS_ATTRIBUTE_CALLBACK['module']),
            CAS_ATTRIBUTE_CALLBACK['function']
        )


################ SECURE AUTH ITEMS ###############################
# Secret things: passwords, access keys, etc.
with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.json") as auth_file:
    AUTH_TOKENS = json.load(auth_file)

EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', EMAIL_HOST_USER)
EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', EMAIL_HOST_PASSWORD)

# If Segment.io key specified, load it and turn on Segment.io if the feature flag is set
# Note that this is the Studio key. There is a separate key for the LMS.
SEGMENT_IO_KEY = AUTH_TOKENS.get('SEGMENT_IO_KEY')
if SEGMENT_IO_KEY:
    FEATURES['SEGMENT_IO'] = ENV_TOKENS.get('SEGMENT_IO', False)

AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"]
if AWS_ACCESS_KEY_ID == "":
    AWS_ACCESS_KEY_ID = None

AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"]
if AWS_SECRET_ACCESS_KEY == "":
    AWS_SECRET_ACCESS_KEY = None

DATABASES = AUTH_TOKENS['DATABASES']
MODULESTORE = AUTH_TOKENS['MODULESTORE']
CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
# Datadog for events!
DATADOG = AUTH_TOKENS.get("DATADOG", {})
DATADOG.update(ENV_TOKENS.get("DATADOG", {}))

# TODO: deprecated (compatibility with previous settings)
if 'DATADOG_API' in AUTH_TOKENS:
    DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']

# Celery Broker
CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "")
CELERY_BROKER_VHOST = ENV_TOKENS.get("CELERY_BROKER_VHOST", "")
CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "")
CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "")

BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT,
                                            CELERY_BROKER_USER,
                                            CELERY_BROKER_PASSWORD,
                                            CELERY_BROKER_HOSTNAME,
                                            CELERY_BROKER_VHOST)

# Event tracking
TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {}))
EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {}))

SUBDOMAIN_BRANDING = ENV_TOKENS.get('SUBDOMAIN_BRANDING', {})
VIRTUAL_UNIVERSITIES = ENV_TOKENS.get('VIRTUAL_UNIVERSITIES', [])

##### ACCOUNT LOCKOUT DEFAULT PARAMETERS #####
MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = ENV_TOKENS.get("MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", 5)
MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = ENV_TOKENS.get("MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", 15 * 60)

MICROSITE_CONFIGURATION = ENV_TOKENS.get('MICROSITE_CONFIGURATION', {})
MICROSITE_ROOT_DIR = path(ENV_TOKENS.get('MICROSITE_ROOT_DIR', ''))

#### PASSWORD POLICY SETTINGS #####
PASSWORD_MIN_LENGTH = ENV_TOKENS.get("PASSWORD_MIN_LENGTH")
PASSWORD_MAX_LENGTH = ENV_TOKENS.get("PASSWORD_MAX_LENGTH")
PASSWORD_COMPLEXITY = ENV_TOKENS.get("PASSWORD_COMPLEXITY", {})
PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = ENV_TOKENS.get("PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD")
PASSWORD_DICTIONARY = ENV_TOKENS.get("PASSWORD_DICTIONARY", [])

### INACTIVITY SETTINGS ####
SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = AUTH_TOKENS.get("SESSION_INACTIVITY_TIMEOUT_IN_SECONDS")

##### X-Frame-Options response header settings #####
X_FRAME_OPTIONS = ENV_TOKENS.get('X_FRAME_OPTIONS', X_FRAME_OPTIONS)

##### ADVANCED_SECURITY_CONFIG #####
ADVANCED_SECURITY_CONFIG = ENV_TOKENS.get('ADVANCED_SECURITY_CONFIG', {})

########NEW FILE########
__FILENAME__ = aws_migrate
"""
A Django settings file for use on AWS while running
database migrations, since we don't want to normally run the
LMS with enough privileges to modify the database schema.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

# Import everything from .aws so that our settings are based on those.
from .aws import *
import os
from django.core.exceptions import ImproperlyConfigured

DB_OVERRIDES = dict(
    PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
    ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
    USER=os.environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
    NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
    HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
    PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
)

if DB_OVERRIDES['PASSWORD'] is None:
    raise ImproperlyConfigured("No database password was provided for running "
                               "migrations.  This is fatal.")

for override, value in DB_OVERRIDES.iteritems():
    DATABASES['default'][override] = value

########NEW FILE########
__FILENAME__ = bok_choy
"""
Settings for bok choy tests
"""

import os
from path import path


########################## Prod-like settings ###################################
# These should be as close as possible to the settings we use in production.
# As in prod, we read in environment and auth variables from JSON files.
# Unlike in prod, we use the JSON files stored in this repo.
# This is a convenience for ensuring (a) that we can consistently find the files
# and (b) that the files are the same in Jenkins as in local dev.
os.environ['SERVICE_VARIANT'] = 'bok_choy'
os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname()  # pylint: disable=E1120

from .aws import *  # pylint: disable=W0401, W0614

######################### Testing overrides ####################################

# Needed for the reset database management command
INSTALLED_APPS += ('django_extensions',)

# Redirect to the test_root folder within the repo
TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=E1120
GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
LOG_DIR = (TEST_ROOT / "log").abspath()

# Configure Mongo modulestore to use the test folder within the repo
for store in ["default", "direct"]:
    MODULESTORE[store]['OPTIONS']['fs_root'] = (TEST_ROOT / "data").abspath()  # pylint: disable=E1120

# Enable django-pipeline and staticfiles
STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
PIPELINE = True

# Silence noisy logs
import logging
LOG_OVERRIDES = [
    ('track.middleware', logging.CRITICAL),
    ('edx.discussion', logging.CRITICAL),
]
for log_name, log_level in LOG_OVERRIDES:
    logging.getLogger(log_name).setLevel(log_level)

# Use the auto_auth workflow for creating users and logging them in
FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True

# Unfortunately, we need to use debug mode to serve staticfiles
DEBUG = True

# Point the URL used to test YouTube availability to our stub YouTube server
YOUTUBE_PORT = 9080
YOUTUBE['API'] = "127.0.0.1:{0}/get_youtube_api/".format(YOUTUBE_PORT)
YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)

########NEW FILE########
__FILENAME__ = common
# -*- coding: utf-8 -*-
"""
This is the common settings file, intended to set sane defaults. If you have a
piece of configuration that's dependent on a set of feature flags being set,
then create a function that returns the calculated value based on the value of
FEATURES[...]. Modules that extend this one can change the feature
configuration in an environment specific config file and re-calculate those
values.

We should make a method that calls all these config methods so that you just
make one call at the end of your site-specific dev file to reset all the
dependent variables (like INSTALLED_APPS) for you.

Longer TODO:
1. Right now our treatment of static content in general and in particular
   course-specific static content is haphazard.
2. We should have a more disciplined approach to feature flagging, even if it
   just means that we stick them in a dict called FEATURES.
3. We need to handle configuration for multiple courses. This could be as
   multiple sites, but we do need a way to map their data assets.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0611, W0614

import imp
import sys
import lms.envs.common
from lms.envs.common import (
    USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED
)
from path import path

from lms.lib.xblock.mixin import LmsBlockMixin
from cms.lib.xblock.mixin import CmsBlockMixin
from dealer.git import git

############################ FEATURE CONFIGURATION #############################

FEATURES = {
    'USE_DJANGO_PIPELINE': True,

    'GITHUB_PUSH': False,

    # for consistency in user-experience, keep the value of the following 3 settings
    # in sync with the ones in lms/envs/common.py
    'ENABLE_DISCUSSION_SERVICE': True,
    'ENABLE_TEXTBOOK': True,
    'ENABLE_STUDENT_NOTES': True,

    'AUTH_USE_CERTIFICATES': False,

    # email address for studio staff (eg to request course creation)
    'STUDIO_REQUEST_EMAIL': '',

    'STUDIO_NPS_SURVEY': True,

    # Segment.io - must explicitly turn it on for production
    'SEGMENT_IO': False,

    # Enable URL that shows information about the status of various services
    'ENABLE_SERVICE_STATUS': False,

    # Don't autoplay videos for course authors
    'AUTOPLAY_VIDEOS': False,

    # If set to True, new Studio users won't be able to author courses unless
    # edX has explicitly added them to the course creator group.
    'ENABLE_CREATOR_GROUP': False,

    # whether to use password policy enforcement or not
    'ENFORCE_PASSWORD_POLICY': False,

    # If set to True, Studio won't restrict the set of advanced components
    # to just those pre-approved by edX
    'ALLOW_ALL_ADVANCED_COMPONENTS': False,

    # Turn off account locking if failed login attempts exceeds a limit
    'ENABLE_MAX_FAILED_LOGIN_ATTEMPTS': False,

    # Allow editing of short description in course settings in cms
    'EDITABLE_SHORT_DESCRIPTION': True,

    # Hide any Personally Identifiable Information from application logs
    'SQUELCH_PII_IN_LOGS': False,

    # Toggles the embargo functionality, which enable embargoing for particular courses
    'EMBARGO': False,

    # Toggles the embargo site functionality, which enable embargoing for the whole site
    'SITE_EMBARGOED': False,

    # Turn on/off Microsites feature
    'USE_MICROSITES': False,

    # Allow creating courses with non-ascii characters in the course id
    'ALLOW_UNICODE_COURSE_ID': False,

    # Prevent concurrent logins per user
    'PREVENT_CONCURRENT_LOGINS': False,

    # Turn off Advanced Security by default
    'ADVANCED_SECURITY': False,
}
ENABLE_JASMINE = False


############################# SET PATH INFORMATION #############################
PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /edx-platform/cms
REPO_ROOT = PROJECT_ROOT.dirname()
COMMON_ROOT = REPO_ROOT / "common"
LMS_ROOT = REPO_ROOT / "lms"
ENV_ROOT = REPO_ROOT.dirname()  # virtualenv dir /edx-platform is in

GITHUB_REPO_ROOT = ENV_ROOT / "data"

sys.path.append(REPO_ROOT)
sys.path.append(PROJECT_ROOT / 'djangoapps')
sys.path.append(COMMON_ROOT / 'djangoapps')
sys.path.append(COMMON_ROOT / 'lib')

# For geolocation ip database
GEOIP_PATH = REPO_ROOT / "common/static/data/geoip/GeoIP.dat"


############################# WEB CONFIGURATION #############################
# This is where we stick our compiled template files.
from tempdir import mkdtemp_clean
MAKO_MODULE_DIR = mkdtemp_clean('mako')
MAKO_TEMPLATES = {}
MAKO_TEMPLATES['main'] = [
    PROJECT_ROOT / 'templates',
    COMMON_ROOT / 'templates',
    COMMON_ROOT / 'djangoapps' / 'pipeline_mako' / 'templates',
    COMMON_ROOT / 'djangoapps' / 'pipeline_js' / 'templates',
]

for namespace, template_dirs in lms.envs.common.MAKO_TEMPLATES.iteritems():
    MAKO_TEMPLATES['lms.' + namespace] = template_dirs

TEMPLATE_DIRS = MAKO_TEMPLATES['main']

EDX_ROOT_URL = ''

LOGIN_REDIRECT_URL = EDX_ROOT_URL + '/signin'
LOGIN_URL = EDX_ROOT_URL + '/signin'


TEMPLATE_CONTEXT_PROCESSORS = (
    'django.core.context_processors.request',
    'django.core.context_processors.static',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.i18n',
    'django.contrib.auth.context_processors.auth',  # this is required for admin
    'django.core.context_processors.csrf',
    'dealer.contrib.django.staff.context_processor',  # access git revision
    'contentstore.context_processors.doc_url',
)

# use the ratelimit backend to prevent brute force attacks
AUTHENTICATION_BACKENDS = (
    'ratelimitbackend.backends.RateLimitModelBackend',
)

LMS_BASE = None

#################### CAPA External Code Evaluation #############################
XQUEUE_INTERFACE = {
    'url': 'http://localhost:8888',
    'django_auth': {'username': 'local',
                    'password': 'local'},
    'basic_auth': None,
}


################################# Middleware ###################################
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'staticfiles.finders.FileSystemFinder',
    'staticfiles.finders.AppDirectoriesFinder',
    'pipeline.finders.PipelineFinder',
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'request_cache.middleware.RequestCache',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'method_override.middleware.MethodOverrideMiddleware',

    # Instead of AuthenticationMiddleware, we use a cache-backed version
    'cache_toolbox.middleware.CacheBackedAuthenticationMiddleware',
    'student.middleware.UserStandingMiddleware',
    'contentserver.middleware.StaticContentServer',
    'crum.CurrentRequestUserMiddleware',

    'django.contrib.messages.middleware.MessageMiddleware',
    'track.middleware.TrackMiddleware',

    # Allows us to dark-launch particular languages
    'dark_lang.middleware.DarkLangMiddleware',

    'embargo.middleware.EmbargoMiddleware',

    # Detects user-requested locale from 'accept-language' header in http request
    'django.middleware.locale.LocaleMiddleware',

    'django.middleware.transaction.TransactionMiddleware',
    # needs to run after locale middleware (or anything that modifies the request context)
    'edxmako.middleware.MakoMiddleware',

    # catches any uncaught RateLimitExceptions and returns a 403 instead of a 500
    'ratelimitbackend.middleware.RateLimitMiddleware',

    # for expiring inactive sessions
    'session_inactivity_timeout.middleware.SessionInactivityTimeout',

    # use Django built in clickjacking protection
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

# Clickjacking protection can be enabled by setting this to 'DENY'
X_FRAME_OPTIONS = 'ALLOW'

############# XBlock Configuration ##########

# Import after sys.path fixup
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore import prefer_xmodules
from xmodule.x_module import XModuleMixin

# This should be moved into an XBlock Runtime/Application object
# once the responsibility of XBlock creation is moved out of modulestore - cpennington
XBLOCK_MIXINS = (LmsBlockMixin, CmsBlockMixin, InheritanceMixin, XModuleMixin)

# Allow any XBlock in Studio
# You should also enable the ALLOW_ALL_ADVANCED_COMPONENTS feature flag, so that
# xblocks can be added via advanced settings
XBLOCK_SELECT_FUNCTION = prefer_xmodules

############################ DJANGO_BUILTINS ################################
# Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
DEBUG = False
TEMPLATE_DEBUG = False

# Site info
SITE_ID = 1
SITE_NAME = "localhost:8001"
HTTPS = 'on'
ROOT_URLCONF = 'cms.urls'

# Email
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
EMAIL_HOST = 'localhost'
EMAIL_PORT = 25
EMAIL_USE_TLS = False
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
DEFAULT_FROM_EMAIL = 'registration@example.com'
DEFAULT_FEEDBACK_EMAIL = 'feedback@example.com'
SERVER_EMAIL = 'devops@example.com'
ADMINS = ()
MANAGERS = ADMINS

# Static content
STATIC_URL = '/static/' + git.revision + "/"
ADMIN_MEDIA_PREFIX = '/static/admin/'
STATIC_ROOT = ENV_ROOT / "staticfiles" / git.revision

STATICFILES_DIRS = [
    COMMON_ROOT / "static",
    PROJECT_ROOT / "static",
    LMS_ROOT / "static",

    # This is how you would use the textbook images locally
    # ("book", ENV_ROOT / "book_images"),
]

# Locale/Internationalization
TIME_ZONE = 'America/New_York'  # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
LANGUAGE_CODE = 'en'  # http://www.i18nguy.com/unicode/language-identifiers.html

LANGUAGES = lms.envs.common.LANGUAGES
USE_I18N = True
USE_L10N = True

# Localization strings (e.g. django.po) are under this directory
LOCALE_PATHS = (REPO_ROOT + '/conf/locale',)  # edx-platform/conf/locale/

# Messages
MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'

##### EMBARGO #####
EMBARGO_SITE_REDIRECT_URL = None

############################### Pipeline #######################################

STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'

from rooted_paths import rooted_glob

PIPELINE_CSS = {
    'style-vendor': {
        'source_filenames': [
            'css/vendor/normalize.css',
            'css/vendor/font-awesome.css',
            'css/vendor/html5-input-polyfills/number-polyfill.css',
            'js/vendor/CodeMirror/codemirror.css',
            'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css',
            'css/vendor/jquery.qtip.min.css',
            'js/vendor/markitup/skins/simple/style.css',
            'js/vendor/markitup/sets/wiki/style.css',
        ],
        'output_filename': 'css/cms-style-vendor.css',
    },
    'style-vendor-tinymce-content': {
        'source_filenames': [
            'css/tinymce-studio-content-fonts.css',
            'js/vendor/tinymce/js/tinymce/skins/studio-tmce4/content.min.css',
            'css/tinymce-studio-content.css'
        ],
        'output_filename': 'css/cms-style-vendor-tinymce-content.css',
    },
    'style-vendor-tinymce-skin': {
        'source_filenames': [
            'js/vendor/tinymce/js/tinymce/skins/studio-tmce4/skin.min.css'
        ],
        'output_filename': 'css/cms-style-vendor-tinymce-skin.css',
    },
    'style-app': {
        'source_filenames': [
            'sass/style-app.css',
        ],
        'output_filename': 'css/cms-style-app.css',
    },
    'style-app-extend1': {
        'source_filenames': [
            'sass/style-app-extend1.css',
        ],
        'output_filename': 'css/cms-style-app-extend1.css',
    },
    'style-xmodule': {
        'source_filenames': [
            'sass/style-xmodule.css',
        ],
        'output_filename': 'css/cms-style-xmodule.css',
    },
}

# test_order: Determines the position of this chunk of javascript on
# the jasmine test page
PIPELINE_JS = {
    'module-js': {
        'source_filenames': (
            rooted_glob(COMMON_ROOT / 'static/', 'xmodule/descriptors/js/*.js') +
            rooted_glob(COMMON_ROOT / 'static/', 'xmodule/modules/js/*.js') +
            rooted_glob(COMMON_ROOT / 'static/', 'coffee/src/discussion/*.js')
        ),
        'output_filename': 'js/cms-modules.js',
        'test_order': 1
    },
}

PIPELINE_COMPILERS = (
    'pipeline.compilers.coffee.CoffeeScriptCompiler',
)

PIPELINE_CSS_COMPRESSOR = None
PIPELINE_JS_COMPRESSOR = None

STATICFILES_IGNORE_PATTERNS = (
    "*.py",
    "*.pyc"
    # it would be nice if we could do, for example, "**/*.scss",
    # but these strings get passed down to the `fnmatch` module,
    # which doesn't support that. :(
    # http://docs.python.org/2/library/fnmatch.html
    "sass/*.scss",
    "sass/*/*.scss",
    "sass/*/*/*.scss",
    "sass/*/*/*/*.scss",
    "coffee/*.coffee",
    "coffee/*/*.coffee",
    "coffee/*/*/*.coffee",
    "coffee/*/*/*/*.coffee",

    # Symlinks used by js-test-tool
    "xmodule_js",
    "common_static",
)

PIPELINE_YUI_BINARY = 'yui-compressor'

################################# CELERY ######################################

# Message configuration

CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

CELERY_MESSAGE_COMPRESSION = 'gzip'

# Results configuration

CELERY_IGNORE_RESULT = False
CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True

# Events configuration

CELERY_TRACK_STARTED = True

CELERY_SEND_EVENTS = True
CELERY_SEND_TASK_SENT_EVENT = True

# Exchange configuration

CELERY_DEFAULT_EXCHANGE = 'edx.core'
CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'

# Queues configuration

HIGH_PRIORITY_QUEUE = 'edx.core.high'
DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
LOW_PRIORITY_QUEUE = 'edx.core.low'

CELERY_QUEUE_HA_POLICY = 'all'

CELERY_CREATE_MISSING_QUEUES = True

CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE

CELERY_QUEUES = {
    HIGH_PRIORITY_QUEUE: {},
    LOW_PRIORITY_QUEUE: {},
    DEFAULT_PRIORITY_QUEUE: {}
}


############################## Video ##########################################

YOUTUBE = {
    # YouTube JavaScript API
    'API': 'www.youtube.com/iframe_api',

    # URL to test YouTube availability
    'TEST_URL': 'gdata.youtube.com/feeds/api/videos/',

    # Current youtube api for requesting transcripts.
    # For example: http://video.google.com/timedtext?lang=en&v=j_jEn79vS3g.
    'TEXT_API': {
        'url': 'video.google.com/timedtext',
        'params': {
            'lang': 'en',
            'v': 'set_youtube_id_of_11_symbols_here',
        },
    },
}

############################ APPS #####################################

INSTALLED_APPS = (
    # Standard apps
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'djcelery',
    'south',
    'method_override',

    # Database-backed configuration
    'config_models',

    # Monitor the status of services
    'service_status',

    # Testing
    'django_nose',

    # For CMS
    'contentstore',
    'course_creators',
    'student',  # misleading name due to sharing with lms
    'course_groups',  # not used in cms (yet), but tests run

    # Tracking
    'track',
    'eventtracking.django',

    # Monitoring
    'datadog',

    # For asset pipelining
    'edxmako',
    'pipeline',
    'staticfiles',
    'static_replace',

    # comment common
    'django_comment_common',

    # for course creator table
    'django.contrib.admin',

    # for managing course modes
    'course_modes',

    # Dark-launching languages
    'dark_lang',

    # Student identity reverification
    'reverification',

    # User preferences
    'user_api',
    'django_openid_auth',

    'embargo',

    # Monitoring signals
    'monitoring',
)


################# EDX MARKETING SITE ##################################

EDXMKTG_COOKIE_NAME = 'edxloggedin'
MKTG_URLS = {}
MKTG_URL_LINK_MAP = {

}

COURSES_WITH_UNSAFE_CODE = []

############################## EVENT TRACKING #################################

TRACK_MAX_EVENT = 50000

TRACKING_BACKENDS = {
    'logger': {
        'ENGINE': 'track.backends.logger.LoggerBackend',
        'OPTIONS': {
            'name': 'tracking'
        }
    }
}

# We're already logging events, and we don't want to capture user
# names/passwords.  Heartbeat events are likely not interesting.
TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat']

EVENT_TRACKING_ENABLED = True
EVENT_TRACKING_BACKENDS = {
    'logger': {
        'ENGINE': 'eventtracking.backends.logger.LoggerBackend',
        'OPTIONS': {
            'name': 'tracking',
            'max_event_size': TRACK_MAX_EVENT,
        }
    }
}
EVENT_TRACKING_PROCESSORS = [
    {
        'ENGINE': 'track.shim.LegacyFieldMappingProcessor'
    }
]

#### PASSWORD POLICY SETTINGS #####

PASSWORD_MIN_LENGTH = None
PASSWORD_MAX_LENGTH = None
PASSWORD_COMPLEXITY = {}
PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = None
PASSWORD_DICTIONARY = []

##### ACCOUNT LOCKOUT DEFAULT PARAMETERS #####
MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = 5
MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = 15 * 60


### Apps only installed in some instances

OPTIONAL_APPS = (
    'edx_jsdraw',
    'mentoring',

    # edx-ora2
    'submissions',
    'openassessment',
    'openassessment.assessment',
    'openassessment.workflow',
    'openassessment.xblock'
)


for app_name in OPTIONAL_APPS:
    # First attempt to only find the module rather than actually importing it,
    # to avoid circular references - only try to import if it can't be found
    # by find_module, which doesn't work with import hooks
    try:
        imp.find_module(app_name)
    except ImportError:
        try:
            __import__(app_name)
        except ImportError:
            continue
    INSTALLED_APPS += (app_name,)

### ADVANCED_SECURITY_CONFIG
# Empty by default
ADVANCED_SECURITY_CONFIG = {}

########NEW FILE########
__FILENAME__ = debug_upload
# pylint: disable=W0614, W0401
from .dev import *

FILE_UPLOAD_HANDLERS = (
    'contentstore.debug_file_uploader.DebugFileUploader',
    'django.core.files.uploadhandler.MemoryFileUploadHandler',
    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
)

########NEW FILE########
__FILENAME__ = dev
"""
This config file runs the simplest dev environment"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .common import *
from logsettings import get_logger_config

# import settings from LMS for consistent behavior with CMS
from lms.envs.dev import (WIKI_ENABLED)

DEBUG = True
TEMPLATE_DEBUG = DEBUG
LOGGING = get_logger_config(ENV_ROOT / "log",
                            logging_env="dev",
                            tracking_filename="tracking.log",
                            dev_env=True,
                            debug=True)

modulestore_options = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'fs_root': GITHUB_REPO_ROOT,
    'render_template': 'edxmako.shortcuts.render_to_string',
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': modulestore_options
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': modulestore_options
    },
    'split': {
        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': modulestore_options
    }
}


# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)
CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'DOC_STORE_CONFIG': {
        'host': 'localhost',
        'db': 'xcontent',
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': ENV_ROOT / "db" / "edx.db",
    }
}

LMS_BASE = "localhost:8000"
FEATURES['PREVIEW_LMS_BASE'] = "localhost:8000"

REPOS = {
    'edx4edx': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/edx4edx.git',
    },
    'content-mit-6002x': {
        'branch': 'master',
        # 'origin': 'git@github.com:MITx/6002x-fall-2012.git',
        'origin': 'git@github.com:MITx/content-mit-6002x.git',
    },
    '6.00x': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/6.00x.git',
    },
    '7.00x': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/7.00x.git',
    },
    '3.091x': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/3.091x.git',
    },
}

CACHES = {
    # This is the cache used for most things. Askbot will not work without a
    # functioning cache -- it relies on caching to load its settings in places.
    # In staging/prod envs, the sessions also live here.
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_loc_mem_cache',
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    # The general cache is what you get if you use our util.cache. It's used for
    # things like caching the course.xml file for different A/B test groups.
    # We set it to be a DummyCache to force reloading of course.xml in dev.
    # In staging environments, we would grab VERSION from data uploaded by the
    # push process.
    'general': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'KEY_PREFIX': 'general',
        'VERSION': 4,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    'mongo_metadata_inheritance': {
        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        'LOCATION': '/var/tmp/mongo_metadata_inheritance',
        'TIMEOUT': 300,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },
    'loc_cache': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_location_mem_cache',
    },

}

# Make the keyedcache startup warnings go away
CACHE_TIMEOUT = 0

# Dummy secret key for dev
SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'

################################ PIPELINE #################################

PIPELINE_SASS_ARGUMENTS = '--debug-info --require {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)

################################# CELERY ######################################

# By default don't use a worker, execute tasks as if they were local functions
CELERY_ALWAYS_EAGER = True

################################ DEBUG TOOLBAR #################################
INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
INTERNAL_IPS = ('127.0.0.1',)

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',

    #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
    #  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
    #  hit twice). So you can uncomment when you need to diagnose performance
    #  problems, but you shouldn't leave it on.
    #  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
)

DEBUG_TOOLBAR_CONFIG = {
    'INTERCEPT_REDIRECTS': False
}

# To see stacktraces for MongoDB queries, set this to True.
# Stacktraces slow down page loads drastically (for pages with lots of queries).
DEBUG_TOOLBAR_MONGO_STACKTRACES = False

# disable NPS survey in dev mode
FEATURES['STUDIO_NPS_SURVEY'] = False

# Enable URL that shows information about the status of variuous services
FEATURES['ENABLE_SERVICE_STATUS'] = True

############################# SEGMENT-IO ##################################

# If there's an environment variable set, grab it and turn on Segment.io
# Note that this is the Studio key. There is a separate key for the LMS.
import os
SEGMENT_IO_KEY = os.environ.get('SEGMENT_IO_KEY')
if SEGMENT_IO_KEY:
    FEATURES['SEGMENT_IO'] = True


#####################################################################
# Lastly, see if the developer has any local overrides.
try:
    from .private import *  # pylint: disable=F0401
except ImportError:
    pass

########NEW FILE########
__FILENAME__ = devstack
"""
Specific overrides to the base prod settings to make development easier.
"""

from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import

# Don't use S3 in devstack, fall back to filesystem
del DEFAULT_FILE_STORAGE
MEDIA_ROOT = "/edx/var/edxapp/uploads"

DEBUG = True
USE_I18N = True
TEMPLATE_DEBUG = DEBUG

################################ LOGGERS ######################################

import logging

# Disable noisy loggers
for pkg_name in ['track.contexts', 'track.middleware', 'dd.dogapi']:
    logging.getLogger(pkg_name).setLevel(logging.CRITICAL)


################################ EMAIL ########################################

EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

################################# LMS INTEGRATION #############################

LMS_BASE = "localhost:8000"
FEATURES['PREVIEW_LMS_BASE'] = "preview." + LMS_BASE

############################# ADVANCED COMPONENTS #############################

# Make it easier to test advanced components in local dev
FEATURES['ALLOW_ALL_ADVANCED_COMPONENTS'] = True

################################# CELERY ######################################

# By default don't use a worker, execute tasks as if they were local functions
CELERY_ALWAYS_EAGER = True

################################ DEBUG TOOLBAR ################################
INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
INTERNAL_IPS = ('127.0.0.1',)

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',

    #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
    #  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
    #  hit twice). So you can uncomment when you need to diagnose performance
    #  problems, but you shouldn't leave it on.
    #  'debug_toolbar.panels.profiling.ProfilingPanel',
)

DEBUG_TOOLBAR_CONFIG = {
    'INTERCEPT_REDIRECTS': False,
    'SHOW_TOOLBAR_CALLBACK': lambda _: True,
}

# To see stacktraces for MongoDB queries, set this to True.
# Stacktraces slow down page loads drastically (for pages with lots of queries).
DEBUG_TOOLBAR_MONGO_STACKTRACES = False

###############################################################################
# Lastly, see if the developer has any local overrides.
try:
    from .private import *  # pylint: disable=F0401
except ImportError:
    pass

########NEW FILE########
__FILENAME__ = dev_dbperf
"""
This configuration is to turn on the Django Toolbar stats for DB access stats, for performance analysis
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .dev import *

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',
    'debug_toolbar_mongo.panel.MongoDebugPanel'

    #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
    #  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
    #  hit twice). So you can uncomment when you need to diagnose performance
    #  problems, but you shouldn't leave it on.
    #  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
)

# To see stacktraces for MongoDB queries, set this to True.
# Stacktraces slow down page loads drastically (for pages with lots of queries).
DEBUG_TOOLBAR_MONGO_STACKTRACES = True

########NEW FILE########
__FILENAME__ = dev_ike
# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

# dev environment for ichuang/mit

# FORCE_SCRIPT_NAME = '/cms'

from .common import *
from .dev import *

FEATURES['AUTH_USE_CERTIFICATES'] = True

FEATURES['USE_DJANGO_PIPELINE'] = False      # don't recompile scss

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')  	# django 1.4 for nginx ssl proxy

########NEW FILE########
__FILENAME__ = dev_shared_preview
"""
This configuration is have localdev use a preview.localhost hostname for the preview LMS so that we can share
the same process between preview and published
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .dev import *

FEATURES['PREVIEW_LMS_BASE'] = "preview.localhost:8000"

########NEW FILE########
__FILENAME__ = dev_with_worker
"""
This config file follows the dev enviroment, but adds the
requirement of a celery worker running in the background to process
celery tasks.

The worker can be executed using:

django_admin.py celery worker
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from dev import *

################################# CELERY ######################################

# Requires a separate celery worker

CELERY_ALWAYS_EAGER = False

# Use django db as the broker and result store

BROKER_URL = 'django://'
INSTALLED_APPS += ('djcelery.transport', )
CELERY_RESULT_BACKEND = 'database'
DJKOMBU_POLLING_INTERVAL = 1.0

# Disable transaction management because we are using a worker. Views
# that request a task and wait for the result will deadlock otherwise.

MIDDLEWARE_CLASSES = tuple(
    c for c in MIDDLEWARE_CLASSES
    if c != 'django.middleware.transaction.TransactionMiddleware')

# Note: other alternatives for disabling transactions don't work in 1.4
# https://code.djangoproject.com/ticket/2304
# https://code.djangoproject.com/ticket/16039

########NEW FILE########
__FILENAME__ = microsite_test
"""
This is a localdev test for the Microsite processing pipeline
"""
# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .dev import *

MICROSITE_NAMES = ['openedx']
MICROSITE_CONFIGURATION = {}
FEATURES['USE_MICROSITES'] = True

########NEW FILE########
__FILENAME__ = test
"""
This config file runs the simplest dev environment using sqlite, and db-based
sessions. Assumes structure:

/envroot/
        /db   # This is where it'll write the database file
        /edx-platform  # The location of this repo
        /log  # Where we're going to write log files
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .common import *
import os
from path import path
from warnings import filterwarnings

# import settings from LMS for consistent behavior with CMS
from lms.envs.test import (WIKI_ENABLED)

# Nose Test Runner
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

_system = 'cms'
_report_dir = REPO_ROOT / 'reports' / _system
_report_dir.makedirs_p()

NOSE_ARGS = [
    '--id-file', REPO_ROOT / '.testids' / _system / 'noseids',
    '--xunit-file', _report_dir / 'nosetests.xml',
]

TEST_ROOT = path('test_root')

# Want static files in the same dir for running on jenkins.
STATIC_ROOT = TEST_ROOT / "staticfiles"

GITHUB_REPO_ROOT = TEST_ROOT / "data"
COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"

# For testing "push to lms"
FEATURES['ENABLE_EXPORT_GIT'] = True
GIT_REPO_EXPORT_DIR = TEST_ROOT / "export_course_repos"

# Makes the tests run much faster...
SOUTH_TESTS_MIGRATE = False  # To disable migrations and use syncdb instead

# TODO (cpennington): We need to figure out how envs/test.py can inject things into common.py so that we don't have to repeat this sort of thing
STATICFILES_DIRS = [
    COMMON_ROOT / "static",
    PROJECT_ROOT / "static",
]
STATICFILES_DIRS += [
    (course_dir, COMMON_TEST_DATA_ROOT / course_dir)
    for course_dir in os.listdir(COMMON_TEST_DATA_ROOT)
    if os.path.isdir(COMMON_TEST_DATA_ROOT / course_dir)
]

DOC_STORE_CONFIG = {
    'host': 'localhost',
    'db': 'test_xmodule',
    'collection': 'test_modulestore',
}

MODULESTORE_OPTIONS = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'fs_root': TEST_ROOT / "data",
    'render_template': 'edxmako.shortcuts.render_to_string',
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'draft': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'split': {
        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    }
}

CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'DOC_STORE_CONFIG': {
        'host': 'localhost',
        'db': 'test_xcontent',
        'collection': 'dont_trip',
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': TEST_ROOT / "db" / "cms.db",
    },
}

LMS_BASE = "localhost:8000"
FEATURES['PREVIEW_LMS_BASE'] = "preview"

CACHES = {
    # This is the cache used for most things. Askbot will not work without a
    # functioning cache -- it relies on caching to load its settings in places.
    # In staging/prod envs, the sessions also live here.
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_loc_mem_cache',
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    # The general cache is what you get if you use our util.cache. It's used for
    # things like caching the course.xml file for different A/B test groups.
    # We set it to be a DummyCache to force reloading of course.xml in dev.
    # In staging environments, we would grab VERSION from data uploaded by the
    # push process.
    'general': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'KEY_PREFIX': 'general',
        'VERSION': 4,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    'mongo_metadata_inheritance': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': '/var/tmp/mongo_metadata_inheritance',
        'TIMEOUT': 300,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },
    'loc_cache': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_location_mem_cache',
    },

}

# Add external_auth to Installed apps for testing
INSTALLED_APPS += ('external_auth', )

# hide ratelimit warnings while running tests
filterwarnings('ignore', message='No request passed to the backend, unable to rate-limit')


################################# CELERY ######################################

CELERY_ALWAYS_EAGER = True
CELERY_RESULT_BACKEND = 'cache'
BROKER_TRANSPORT = 'memory'


########################### Server Ports ###################################

# These ports are carefully chosen so that if the browser needs to
# access them, they will be available through the SauceLabs SSH tunnel
LETTUCE_SERVER_PORT = 8003
XQUEUE_PORT = 8040
YOUTUBE_PORT = 8031
LTI_PORT = 8765
VIDEO_SOURCE_PORT = 8777


################### Make tests faster
# http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/
PASSWORD_HASHERS = (
    'django.contrib.auth.hashers.SHA1PasswordHasher',
    'django.contrib.auth.hashers.MD5PasswordHasher',
)

# dummy segment-io key
SEGMENT_IO_KEY = '***REMOVED***'

# disable NPS survey in test mode
FEATURES['STUDIO_NPS_SURVEY'] = False

FEATURES['ENABLE_SERVICE_STATUS'] = True

# This is to disable a test under the common directory that will not pass when run under CMS
FEATURES['DISABLE_RESET_EMAIL_TEST'] = True

# Toggles embargo on for testing
FEATURES['EMBARGO'] = True

# set up some testing for microsites
MICROSITE_CONFIGURATION = {
    "test_microsite": {
        "domain_prefix": "testmicrosite",
        "university": "test_microsite",
        "platform_name": "Test Microsite",
        "logo_image_url": "test_microsite/images/header-logo.png",
        "email_from_address": "test_microsite@edx.org",
        "payment_support_email": "test_microsite@edx.org",
        "ENABLE_MKTG_SITE": False,
        "SITE_NAME": "test_microsite.localhost",
        "course_org_filter": "TestMicrositeX",
        "course_about_show_social_links": False,
        "css_overrides_file": "test_microsite/css/test_microsite.css",
        "show_partners": False,
        "show_homepage_promo_video": False,
        "course_index_overlay_text": "This is a Test Microsite Overlay Text.",
        "course_index_overlay_logo_file": "test_microsite/images/header-logo.png",
        "homepage_overlay_html": "<h1>This is a Test Microsite Overlay HTML</h1>"
    },
    "default": {
        "university": "default_university",
        "domain_prefix": "www",
    }
}
MICROSITE_ROOT_DIR = COMMON_ROOT / 'test' / 'test_microsites'
FEATURES['USE_MICROSITES'] = True

# For consistency in user-experience, keep the value of this setting in sync with
# the one in lms/envs/test.py
FEATURES['ENABLE_DISCUSSION_SERVICE'] = False

########NEW FILE########
__FILENAME__ = mixin
"""
Mixin defining common Studio functionality
"""

import datetime

from xblock.fields import Scope, Field, Integer, XBlockMixin


class DateTuple(Field):
    """
    Field that stores datetime objects as time tuples
    """
    def from_json(self, value):
        return datetime.datetime(*value[0:6])

    def to_json(self, value):
        if value is None:
            return None

        return list(value.timetuple())


class CmsBlockMixin(XBlockMixin):
    """
    Mixin with fields common to all blocks in Studio
    """
    published_date = DateTuple(help="Date when the module was published", scope=Scope.settings)
    published_by = Integer(help="Id of the user who published this module", scope=Scope.settings)

########NEW FILE########
__FILENAME__ = runtime
"""
XBlock runtime implementations for edX Studio
"""

from django.core.urlresolvers import reverse


def handler_url(block, handler_name, suffix='', query='', thirdparty=False):
    """
    Handler URL function for Studio
    """

    if thirdparty:
        raise NotImplementedError("edX Studio doesn't support third-party xblock handler urls")

    url = reverse('component_handler', kwargs={
        'usage_key_string': unicode(block.scope_ids.usage_id).encode('utf-8'),
        'handler': handler_name,
        'suffix': suffix,
    }).rstrip('/')

    if query:
        url += '?' + query

    return url


def local_resource_url(block, uri):
    """
    local_resource_url for Studio
    """
    return reverse('xblock_resource_url', kwargs={
        'block_type': block.scope_ids.block_type,
        'uri': uri,
    })

########NEW FILE########
__FILENAME__ = test_runtime
"""
Tests of edX Studio runtime functionality
"""
from urlparse import urlparse

from mock import Mock
from unittest import TestCase
from cms.lib.xblock.runtime import handler_url


class TestHandlerUrl(TestCase):
    """Test the LMS handler_url"""

    def setUp(self):
        self.block = Mock()

    def test_trailing_charecters(self):
        self.assertFalse(handler_url(self.block, 'handler').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler').endswith('/'))

        self.assertFalse(handler_url(self.block, 'handler', 'suffix').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler', 'suffix').endswith('/'))

        self.assertFalse(handler_url(self.block, 'handler', 'suffix', 'query').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler', 'suffix', 'query').endswith('/'))

        self.assertFalse(handler_url(self.block, 'handler', query='query').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler', query='query').endswith('/'))

    def _parsed_query(self, query_string):
        """Return the parsed query string from a handler_url generated with the supplied query_string"""
        return urlparse(handler_url(self.block, 'handler', query=query_string)).query

    def test_query_string(self):
        self.assertIn('foo=bar', self._parsed_query('foo=bar'))
        self.assertIn('foo=bar&baz=true', self._parsed_query('foo=bar&baz=true'))
        self.assertIn('foo&bar&baz', self._parsed_query('foo&bar&baz'))

    def _parsed_path(self, handler_name='handler', suffix=''):
        """Return the parsed path from a handler_url with the supplied handler_name and suffix"""
        return urlparse(handler_url(self.block, handler_name, suffix=suffix)).path

    def test_suffix(self):
        self.assertTrue(self._parsed_path(suffix="foo").endswith('foo'))
        self.assertTrue(self._parsed_path(suffix="foo/bar").endswith('foo/bar'))
        self.assertTrue(self._parsed_path(suffix="/foo/bar").endswith('/foo/bar'))

    def test_handler_name(self):
        self.assertIn('handler1', self._parsed_path('handler1'))
        self.assertIn('handler_a', self._parsed_path('handler_a'))

########NEW FILE########
__FILENAME__ = startup
"""
Module with code executed during Studio startup
"""
from django.conf import settings

# Force settings to run so that the python path is modified
settings.INSTALLED_APPS  # pylint: disable=W0104

from django_startup import autostartup


def run():
    """
    Executed during django startup
    """
    autostartup()

########NEW FILE########
__FILENAME__ = urls
from django.conf import settings
from django.conf.urls import patterns, include, url

# There is a course creators admin table.
from ratelimitbackend import admin
admin.autodiscover()

urlpatterns = patterns('',  # nopep8

    url(r'^transcripts/upload$', 'contentstore.views.upload_transcripts', name='upload_transcripts'),
    url(r'^transcripts/download$', 'contentstore.views.download_transcripts', name='download_transcripts'),
    url(r'^transcripts/check$', 'contentstore.views.check_transcripts', name='check_transcripts'),
    url(r'^transcripts/choose$', 'contentstore.views.choose_transcripts', name='choose_transcripts'),
    url(r'^transcripts/replace$', 'contentstore.views.replace_transcripts', name='replace_transcripts'),
    url(r'^transcripts/rename$', 'contentstore.views.rename_transcripts', name='rename_transcripts'),
    url(r'^transcripts/save$', 'contentstore.views.save_transcripts', name='save_transcripts'),

    url(r'^preview/xblock/(?P<usage_key_string>.*?)/handler/(?P<handler>[^/]*)(?:/(?P<suffix>.*))?$',
        'contentstore.views.preview_handler', name='preview_handler'),

    url(r'^xblock/(?P<usage_key_string>.*?)/handler/(?P<handler>[^/]*)(?:/(?P<suffix>.*))?$',
        'contentstore.views.component_handler', name='component_handler'),

    url(r'^xblock/resource/(?P<block_type>[^/]*)/(?P<uri>.*)$',
        'contentstore.views.xblock.xblock_resource', name='xblock_resource_url'),

    # temporary landing page for a course
    url(r'^edge/(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<coursename>[^/]+)$',
        'contentstore.views.landing', name='landing'),

    url(r'^not_found$', 'contentstore.views.not_found', name='not_found'),
    url(r'^server_error$', 'contentstore.views.server_error', name='server_error'),

    # temporary landing page for edge
    url(r'^edge$', 'contentstore.views.edge', name='edge'),
    # noop to squelch ajax errors
    url(r'^event$', 'contentstore.views.event', name='event'),

    url(r'^xmodule/', include('pipeline_js.urls')),
    url(r'^heartbeat$', include('heartbeat.urls')),

    url(r'^user_api/', include('user_api.urls')),
    url(r'^lang_pref/', include('lang_pref.urls')),
)

# User creation and updating views
urlpatterns += patterns(
    '',

    url(r'^create_account$', 'student.views.create_account', name='create_account'),
    url(r'^activate/(?P<key>[^/]*)$', 'student.views.activate_account', name='activate'),

    # ajax view that actually does the work
    url(r'^login_post$', 'student.views.login_user', name='login_post'),
    url(r'^logout$', 'student.views.logout_user', name='logout'),
    url(r'^embargo$', 'student.views.embargo', name="embargo"),
)

# restful api
urlpatterns += patterns(
    'contentstore.views',

    url(r'^$', 'howitworks', name='homepage'),
    url(r'^howitworks$', 'howitworks'),
    url(r'^signup$', 'signup', name='signup'),
    url(r'^signin$', 'login_page', name='login'),
    url(r'^request_course_creator$', 'request_course_creator'),

    url(r'^course_team/(?P<course_key_string>[^/]+)/(?P<email>.+)?$', 'course_team_handler'),
    url(r'^course_info/(?P<course_key_string>[^/]+)$', 'course_info_handler'),
    url(
        r'^course_info_update/(?P<course_key_string>[^/]+)/(?P<provided_id>\d+)?$',
        'course_info_update_handler'
    ),
    url(r'^course/(?P<course_key_string>[^/]+)?$', 'course_handler', name='course_handler'),
    url(r'^subsection/(?P<usage_key_string>[^/]+)$', 'subsection_handler'),
    url(r'^unit/(?P<usage_key_string>[^/]+)$', 'unit_handler'),
    url(r'^container/(?P<usage_key_string>[^/]+)$', 'container_handler'),
    url(r'^checklists/(?P<course_key_string>[^/]+)/(?P<checklist_index>\d+)?$', 'checklists_handler'),
    url(r'^orphan/(?P<course_key_string>[^/]+)$', 'orphan_handler'),
    url(r'^assets/(?P<course_key_string>[^/]+)/(?P<asset_key_string>.+)?$', 'assets_handler'),
    url(r'^import/(?P<course_key_string>[^/]+)$', 'import_handler'),
    url(r'^import_status/(?P<course_key_string>[^/]+)/(?P<filename>.+)$', 'import_status_handler'),
    url(r'^export/(?P<course_key_string>[^/]+)$', 'export_handler'),
    url(r'^xblock/(?P<usage_key_string>[^/]+)/(?P<view_name>[^/]+)$', 'xblock_view_handler'),
    url(r'^xblock/(?P<usage_key_string>[^/]+)?$', 'xblock_handler'),
    url(r'^tabs/(?P<course_key_string>[^/]+)$', 'tabs_handler'),
    url(r'^settings/details/(?P<course_key_string>[^/]+)$', 'settings_handler'),
    url(r'^settings/grading/(?P<course_key_string>[^/]+)(/)?(?P<grader_index>\d+)?$', 'grading_handler'),
    url(r'^settings/advanced/(?P<course_key_string>[^/]+)$', 'advanced_settings_handler'),
    url(r'^textbooks/(?P<course_key_string>[^/]+)$', 'textbooks_list_handler'),
    url(r'^textbooks/(?P<course_key_string>[^/]+)/(?P<textbook_id>\d[^/]*)$', 'textbooks_detail_handler'),
)

js_info_dict = {
    'domain': 'djangojs',
    # No packages needed, we get LOCALE_PATHS anyway.
    'packages': (),
}

urlpatterns += patterns('',
    # Serve catalog of localized strings to be rendered by Javascript
    url(r'^i18n.js$', 'django.views.i18n.javascript_catalog', js_info_dict),
)


if settings.FEATURES.get('ENABLE_EXPORT_GIT'):
    urlpatterns += (url(r'^export_git/(?P<course_key_string>[^/]+)$',
                        'contentstore.views.export_git', name='export_git'),)

if settings.FEATURES.get('ENABLE_SERVICE_STATUS'):
    urlpatterns += patterns('',
        url(r'^status/', include('service_status.urls')),
    )

if settings.FEATURES.get('AUTH_USE_CAS'):
    urlpatterns += (
        url(r'^cas-auth/login/$', 'external_auth.views.cas_login', name="cas-login"),
        url(r'^cas-auth/logout/$', 'django_cas.views.logout', {'next_page': '/'}, name="cas-logout"),
    )

urlpatterns += patterns('', url(r'^admin/', include(admin.site.urls)),)

# enable automatic login
if settings.FEATURES.get('AUTOMATIC_AUTH_FOR_TESTING'):
    urlpatterns += (
        url(r'^auto_auth$', 'student.views.auto_auth'),
    )

if settings.DEBUG:
    try:
        from .urls_dev import urlpatterns as dev_urlpatterns
        urlpatterns += dev_urlpatterns
    except ImportError:
        pass

# Custom error pages
# pylint: disable=C0103
handler404 = 'contentstore.views.render_404'
handler500 = 'contentstore.views.render_500'

# display error page templates, for testing purposes
urlpatterns += (
    url(r'404', handler404),
    url(r'500', handler500),
)

########NEW FILE########
__FILENAME__ = urls_dev
"""
URLconf for development-only views.
This gets imported by urls.py and added to its URLconf if we are running in
development mode; otherwise, it is ignored.
"""
from django.conf.urls import url

urlpatterns = (
    url(r'^dev_mode$', 'contentstore.views.dev.dev_mode', name='dev_mode'),
    url(r'^template/(?P<template>.+)$', 'contentstore.views.dev.dev_show_template'),
)

########NEW FILE########
__FILENAME__ = wsgi
import os

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.aws")

import cms.startup as startup
startup.run()

# This application object is used by the development server
# as well as any WSGI server configured to use this file.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()


########NEW FILE########
__FILENAME__ = app_settings
from django.conf import settings

# Default cache timeout
CACHE_TOOLBOX_DEFAULT_TIMEOUT = getattr(
    settings,
    'CACHE_TOOLBOX_DEFAULT_TIMEOUT',
    60 * 60 * 24 * 3,
)

########NEW FILE########
__FILENAME__ = core
"""
Core methods
------------

.. autofunction:: cache_toolbox.core.get_instance
.. autofunction:: cache_toolbox.core.delete_instance
.. autofunction:: cache_toolbox.core.instance_key

"""

from django.core.cache import cache
from django.db import DEFAULT_DB_ALIAS

from . import app_settings


def get_instance(model, instance_or_pk, timeout=None, using=None):
    """
    Returns the ``model`` instance with a primary key of ``instance_or_pk``.

    If the data is cached it will be returned from there, otherwise the regular
    Django ORM is queried for this instance and the data stored in the cache.

    If omitted, the timeout value defaults to
    ``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).

    Example::

        >>> get_instance(User, 1) # Cache miss
        <User: lamby>
        >>> get_instance(User, 1) # Cache hit
        <User: lamby>
        >>> User.objects.get(pk=1) == get_instance(User, 1)
        True
    """

    pk = getattr(instance_or_pk, 'pk', instance_or_pk)
    key = instance_key(model, instance_or_pk)
    data = cache.get(key)

    if data is not None:
        try:
            # Try and construct instance from dictionary
            instance = model(pk=pk, **data)

            # Ensure instance knows that it already exists in the database,
            # otherwise we will fail any uniqueness checks when saving the
            # instance.
            instance._state.adding = False

            # Specify database so that instance is setup correctly. We don't
            # namespace cached objects by their origin database, however.
            instance._state.db = using or DEFAULT_DB_ALIAS

            return instance
        except:
            # Error when deserialising - remove from the cache; we will
            # fallback and return the underlying instance
            cache.delete(key)

    # Use the default manager so we are never filtered by a .get_query_set()

#    import logging
#    log = logging.getLogger("tracking")
#    log.info( str(pk) )

    instance = model._default_manager.using(using).get(pk=pk)

    data = {}
    for field in instance._meta.fields:
        # Harmless to save, but saves space in the dictionary - we already know
        # the primary key when we lookup
        if field.primary_key:
            continue

        if field.get_internal_type() == 'FileField':
            # Avoid problems with serializing FileFields
            # by only serializing the file name
            file = getattr(instance, field.attname)
            data[field.attname] = file.name
        else:
            data[field.attname] = getattr(instance, field.attname)

    if timeout is None:
        timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT

    cache.set(key, data, timeout)

    return instance


def delete_instance(model, *instance_or_pk):
    """
    Purges the cache keys for the instances of this model.
    """

    cache.delete_many([instance_key(model, x) for x in instance_or_pk])


def instance_key(model, instance_or_pk):
    """
    Returns the cache key for this (model, instance) pair.
    """

    return '%s.%s:%d' % (
        model._meta.app_label,
        model._meta.module_name,
        getattr(instance_or_pk, 'pk', instance_or_pk),
    )


def set_cached_content(content):
    cache.set(unicode(content.location).encode("utf-8"), content)


def get_cached_content(location):
    return cache.get(unicode(location).encode("utf-8"))


def del_cached_content(location):
    # delete content for the given location, as well as for content with run=None.
    # it's possible that the content could have been cached without knowing the
    # course_key - and so without having the run.
    cache.delete_many(
        [unicode(loc).encode("utf-8") for loc in [location, location.replace(run=None)]]
    )

########NEW FILE########
__FILENAME__ = middleware
"""
Cache-backed ``AuthenticationMiddleware``
-----------------------------------------

``CacheBackedAuthenticationMiddleware`` is an
``django.contrib.auth.middleware.AuthenticationMiddleware`` replacement to
avoid querying the database for a ``User`` instance in each request.

Whilst the built-in ``AuthenticationMiddleware`` mechanism will only obtain the
``User`` instance when it is required, the vast majority of sites will do so on
every page to render "Logged in as 'X'" text as well to evaluate the result of
``user.is_authenticated()`` and ``user.is_superuser`` to provide conditional
functionality.

This middleware eliminates the cost of retrieving this ``User`` instance by
caching it using the ``cache_toolbox`` instance caching mechanisms.

Depending on your average number of queries per page, saving one query per
request can---in aggregate---reduce load on your database. In addition,
avoiding the database entirely for pages can avoid incurring any connection
latency in your environment, resulting in faster page loads for your users.

Saving this data in the cache can also be used as a way of authenticating users
in systems outside of Django that should not access your database.  For
example, a "maintenance mode" page would be able to render a personalised
message without touching the database at all but rather authenticating via the
cache.

``CacheBackedAuthenticationMiddleware`` is ``AUTHENTICATION_BACKENDS`` agnostic.

Implementation
~~~~~~~~~~~~~~

The cache and session backends are still accessed on each request - we are
simply assuming that they are cheaper (or otherwise more preferable) to access
than your database. (In the future, signed cookies may allow us to avoid this
lookup altogether -- whilst we could not safely save ``User.password`` in a
cookie, we could use delayed loading to pull it out when needed.)

Another alternative solution would be to store the attributes in the user's
session instead of in the cache. This would save the cache hit on every request
as all the relevant data would be pulled in one go from the session backend.
However, this has two main disadvantages:

 * Session keys are not deterministic -- after making changes to an
   ``auth_user`` row in the database, you cannot determine the user's session
   key to flush the now out-of-sync data (and doing so would log them out
   anyway).

 * Stores data per-session rather than per-user -- if a user logs in from
   multiple computers the data is duplicated in each session. This problem is
   compounded by most projects wishing to avoid expiring session data as long
   as possible (in addition to storing sessions in persistent stores).

Usage
~~~~~

To use, find ``MIDDLEWARE_CLASSES`` in your ``settings.py`` and replace::

    MIDDLEWARE_CLASSES = [
        ...
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        ...
    ]

with::

    MIDDLEWARE_CLASSES = [
        ...
        'cache_toolbox.middleware.CacheBackedAuthenticationMiddleware',
        ...
    ]

You should confirm you are using a ``SESSION_ENGINE`` that doesn't query the
database for each request. The built-in ``cached_db`` engine is the safest
choice for most environments but you may be happy with the trade-offs of the
``memcached`` backend - see the Django documentation for more details.

"""

from django.contrib.auth import SESSION_KEY
from django.contrib.auth.models import User
from django.contrib.auth.middleware import AuthenticationMiddleware

from .model import cache_model


class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware):
    def __init__(self):
        cache_model(User)

    def process_request(self, request):
        try:
            # Try and construct a User instance from data stored in the cache
            request.user = User.get_cached(request.session[SESSION_KEY])
        except:
            # Fallback to constructing the User from the database.
            super(CacheBackedAuthenticationMiddleware, self).process_request(request)

########NEW FILE########
__FILENAME__ = model
"""
Caching model instances
-----------------------

``cache_model`` adds utility methods to a model to obtain ``ForeignKey``
instances via the cache.

Usage
~~~~~

::

    from django.db import models
    from django.contrib.auth.models import User

    class Foo(models.Model):
        name = models.CharField(length=20)

    cache_model(Foo)

::

    >>> a = Foo.objects.create(name='a')
    >>> a
    <Foo: >
    >>> Foo.get_cached(a.pk) # Cache miss
    <Foo: >
    >>> a = Foo.get_cached(a.pk) # Cache hit
    >>> a.name
    u'a'

Instances returned from ``get_cached`` are real model instances::

    >>> a = Foo.get_cached(a.pk) # Cache hit
    >>> type(a)
    <class '__main__.models.A'>
    >>> a.pk
    1L

Invalidation
~~~~~~~~~~~~

Invalidation is performed automatically upon saving or deleting a ``Foo``
instance::

    >>> a = Foo.objects.create(name='a')
    >>> a.name = 'b'
    >>> a.save()
    >>> a = Foo.get_cached(a.pk)
    >>> a.name
    u'b'
    >>> a.delete()
    >>> a = Foo.get_cached(a.pk)
    ... Foo.DoesNotExist
"""

from django.db.models.signals import post_save, post_delete

from .core import get_instance, delete_instance


def cache_model(model, timeout=None):
    if hasattr(model, 'get_cached'):
        # Already patched
        return

    def clear_cache(sender, instance, *args, **kwargs):
        delete_instance(sender, instance)

    post_save.connect(clear_cache, sender=model, weak=False)
    post_delete.connect(clear_cache, sender=model, weak=False)

    @classmethod
    def get(cls, pk, using=None):
        if pk is None:
            return None
        return get_instance(cls, pk, timeout, using)

    model.get_cached = get

########NEW FILE########
__FILENAME__ = relation
"""
Caching instances via ``related_name``
--------------------------------------

``cache_relation`` adds utility methods to a model to obtain ``related_name``
instances via the cache.

Usage
~~~~~

::

    from django.db import models
    from django.contrib.auth.models import User

    class Foo(models.Model):
        user = models.OneToOneField(
            User,
            primary_key=True,
            related_name='foo',
        )

        name = models.CharField(max_length=20)

    cache_relation(User.foo)

::

    >>> user = User.objects.get(pk=1)
    >>> user.foo_cache # Cache miss - hits the database
    <Foo: >
    >>> user = User.objects.get(pk=1)
    >>> user.foo_cache # Cache hit - no database access
    <Foo: >
    >>> user = User.objects.get(pk=2)
    >>> user.foo # Regular lookup - hits the database
    <Foo: >
    >>> user.foo_cache # Special-case: Will not hit cache or database.
    <Foo: >

Accessing ``user_instance.foo_cache`` (note the "_cache" suffix) will now
obtain the related ``Foo`` instance via the cache. Accessing the original
``user_instance.foo`` attribute will perform the lookup as normal.

Invalidation
~~~~~~~~~~~~

Upon saving (or deleting) the instance, the cache is cleared. For example::

    >>> user = User.objects.get(pk=1)
    >>> foo = user.foo_cache # (Assume cache hit from previous session)
    >>> foo.name = "New name"
    >>> foo.save() # Cache is cleared on save
    >>> user = User.objects.get(pk=1)
    >>> user.foo_cache # Cache miss.
    <Foo: >

Manual invalidation may also be performed using the following methods::

    >>> user_instance.foo_cache_clear()
    >>> User.foo_cache_clear_fk(user_instance_pk)

Manual invalidation is required if you use ``.update()`` methods which the
``post_save`` and ``post_delete`` hooks cannot intercept.

Support
~~~~~~~

``cache_relation`` currently only works with ``OneToOneField`` fields. Support
for regular ``ForeignKey`` fields is planned.
"""

from django.db.models.signals import post_save, post_delete

from .core import get_instance, delete_instance


def cache_relation(descriptor, timeout=None):
    rel = descriptor.related
    related_name = '%s_cache' % rel.field.related_query_name()

    @property
    def get(self):
        # Always use the cached "real" instance if available
        try:
            return getattr(self, descriptor.cache_name)
        except AttributeError:
            pass

        # Lookup cached instance
        try:
            return getattr(self, '_%s_cache' % related_name)
        except AttributeError:
            pass

#        import logging
#        log = logging.getLogger("tracking")
#        log.info( "DEBUG: "+str(str(rel.model)+"/"+str(self.pk) ))

        instance = get_instance(rel.model, self.pk, timeout)

        setattr(self, '_%s_cache' % related_name, instance)

        return instance
    setattr(rel.parent_model, related_name, get)

    # Clearing cache

    def clear(self):
        delete_instance(rel.model, self)

    @classmethod
    def clear_pk(cls, *instances_or_pk):
        delete_instance(rel.model, *instances_or_pk)

    def clear_cache(sender, instance, *args, **kwargs):
        delete_instance(rel.model, instance)

    setattr(rel.parent_model, '%s_clear' % related_name, clear)
    setattr(rel.parent_model, '%s_clear_pk' % related_name, clear_pk)

    post_save.connect(clear_cache, sender=rel.model, weak=False)
    post_delete.connect(clear_cache, sender=rel.model, weak=False)

########NEW FILE########
__FILENAME__ = cache_toolbox
from django import template
from django.core.cache import cache
from django.template import Node, TemplateSyntaxError, Variable
from django.template import resolve_variable

register = template.Library()


class CacheNode(Node):
    def __init__(self, nodelist, expire_time, key):
        self.nodelist = nodelist
        self.expire_time = Variable(expire_time)
        self.key = key

    def render(self, context):
        key = resolve_variable(self.key, context)
        expire_time = int(self.expire_time.resolve(context))

        value = cache.get(key)
        if value is None:
            value = self.nodelist.render(context)
            cache.set(key, value, expire_time)
        return value


@register.tag
def cachedeterministic(parser, token):
    """
    This will cache the contents of a template fragment for a given amount of
    time, just like {% cache .. %} except that the key is deterministic and not
    mangled or run through MD5.

    Usage::

        {% cachedeterministic [expire_time] [key] %}
            .. some expensive processing ..
        {% endcachedeterministic %}

    """
    nodelist = parser.parse(('endcachedeterministic',))
    parser.delete_first_token()
    tokens = token.contents.split()
    if len(tokens) != 3:
        raise TemplateSyntaxError(u"'%r' tag requires 2 arguments." % tokens[0])
    return CacheNode(nodelist, tokens[1], tokens[2])


class ShowIfCachedNode(Node):
    def __init__(self, key):
        self.key = key

    def render(self, context):
        key = resolve_variable(self.key, context)
        return cache.get(key) or ''


@register.tag
def showifcached(parser, token):
    """
    Show content if it exists in the cache, otherwise display nothing.

    The key is entirely deterministic and not mangled or run through MD5 (cf.
    {% cache %})

    Usage::

        {% showifcached [key] %}

    """
    tokens = token.contents.split()
    if len(tokens) != 2:
        raise TemplateSyntaxError(u"'%r' tag requires 1 argument." % tokens[0])
    return ShowIfCachedNode(tokens[1])

########NEW FILE########
__FILENAME__ = admin
"""
Admin site models for managing :class:`.ConfigurationModel` subclasses
"""

from django.forms import models
from django.contrib import admin
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse

# pylint: disable=protected-access


class ConfigurationModelAdmin(admin.ModelAdmin):
    """
    :class:`~django.contrib.admin.ModelAdmin` for :class:`.ConfigurationModel` subclasses
    """
    date_hierarchy = 'change_date'

    def get_actions(self, request):
        return {
            'revert': (ConfigurationModelAdmin.revert, 'revert', 'Revert to the selected configuration')
        }

    def get_list_display(self, request):
        return self.model._meta.get_all_field_names()

    # Don't allow deletion of configuration
    def has_delete_permission(self, request, obj=None):
        return False

    # Make all fields read-only when editing an object
    def get_readonly_fields(self, request, obj=None):
        if obj:  # editing an existing object
            return self.model._meta.get_all_field_names()
        return self.readonly_fields

    def add_view(self, request, form_url='', extra_context=None):
        # Prepopulate new configuration entries with the value of the current config
        get = request.GET.copy()
        get.update(models.model_to_dict(self.model.current()))
        request.GET = get
        return super(ConfigurationModelAdmin, self).add_view(request, form_url, extra_context)

    # Hide the save buttons in the change view
    def change_view(self, request, object_id, form_url='', extra_context=None):
        extra_context = extra_context or {}
        extra_context['readonly'] = True
        return super(ConfigurationModelAdmin, self).change_view(
            request,
            object_id,
            form_url,
            extra_context=extra_context
        )

    def save_model(self, request, obj, form, change):
        obj.changed_by = request.user
        super(ConfigurationModelAdmin, self).save_model(request, obj, form, change)

    def revert(self, request, queryset):
        """
        Admin action to revert a configuration back to the selected value
        """
        if queryset.count() != 1:
            self.message_user(request, "Please select a single configuration to revert to.")
            return

        target = queryset[0]
        target.id = None
        self.save_model(request, target, None, False)
        self.message_user(request, "Reverted configuration.")

        return HttpResponseRedirect(
            reverse(
                'admin:{}_{}_change'.format(
                    self.model._meta.app_label,
                    self.model._meta.module_name,
                ),
                args=(target.id,),
            )
        )

########NEW FILE########
__FILENAME__ = models
"""
Django Model baseclass for database-backed configuration.
"""
from django.db import models
from django.contrib.auth.models import User
from django.core.cache import get_cache, InvalidCacheBackendError

try:
    cache = get_cache('configuration')  # pylint: disable=invalid-name
except InvalidCacheBackendError:
    from django.core.cache import cache


class ConfigurationModel(models.Model):
    """
    Abstract base class for model-based configuration

    Properties:
        cache_timeout (int): The number of seconds that this configuration
            should be cached
    """

    class Meta(object):  # pylint: disable=missing-docstring
        abstract = True

    # The number of seconds
    cache_timeout = 600

    change_date = models.DateTimeField(auto_now_add=True)
    changed_by = models.ForeignKey(User, editable=False, null=True, on_delete=models.PROTECT)
    enabled = models.BooleanField(default=False)

    def save(self, *args, **kwargs):
        """
        Clear the cached value when saving a new configuration entry
        """
        super(ConfigurationModel, self).save(*args, **kwargs)
        cache.delete(self.cache_key_name())

    @classmethod
    def cache_key_name(cls):
        """Return the name of the key to use to cache the current configuration"""
        return 'configuration/{}/current'.format(cls.__name__)

    @classmethod
    def current(cls):
        """
        Return the active configuration entry, either from cache,
        from the database, or by creating a new empty entry (which is not
        persisted).
        """
        cached = cache.get(cls.cache_key_name())
        if cached is not None:
            return cached

        try:
            current = cls.objects.order_by('-change_date')[0]
        except IndexError:
            current = cls()

        cache.set(cls.cache_key_name(), current, cls.cache_timeout)
        return current

########NEW FILE########
__FILENAME__ = templatetags
"""
Override the submit_row template tag to remove all save buttons from the
admin dashboard change view if the context has readonly marked in it.
"""

from django.contrib.admin.templatetags.admin_modify import register
from django.contrib.admin.templatetags.admin_modify import submit_row as original_submit_row


@register.inclusion_tag('admin/submit_line.html', takes_context=True)
def submit_row(context):
    """
    Overrides 'django.contrib.admin.templatetags.admin_modify.submit_row'.

    Manipulates the context going into that function by hiding all of the buttons
    in the submit row if the key `readonly` is set in the context.
    """
    ctx = original_submit_row(context)

    if context.get('readonly', False):
        ctx.update({
            'show_delete_link': False,
            'show_save_as_new': False,
            'show_save_and_add_another': False,
            'show_save_and_continue': False,
            'show_save': False,
        })
    else:
        return ctx

########NEW FILE########
__FILENAME__ = tests
"""
Tests of ConfigurationModel
"""

from django.contrib.auth.models import User
from django.db import models
from django.test import TestCase

from freezegun import freeze_time

from mock import patch
from config_models.models import ConfigurationModel


class ExampleConfig(ConfigurationModel):
    """
    Test model for testing ``ConfigurationModels``.
    """
    cache_timeout = 300

    string_field = models.TextField()
    int_field = models.IntegerField(default=10)


@patch('config_models.models.cache')
class ConfigurationModelTests(TestCase):
    """
    Tests of ConfigurationModel
    """
    def setUp(self):
        self.user = User()
        self.user.save()

    def test_cache_deleted_on_save(self, mock_cache):
        ExampleConfig(changed_by=self.user).save()
        mock_cache.delete.assert_called_with(ExampleConfig.cache_key_name())

    def test_cache_key_name(self, _mock_cache):
        self.assertEquals(ExampleConfig.cache_key_name(), 'configuration/ExampleConfig/current')

    def test_no_config_empty_cache(self, mock_cache):
        mock_cache.get.return_value = None

        current = ExampleConfig.current()
        self.assertEquals(current.int_field, 10)
        self.assertEquals(current.string_field, '')
        mock_cache.set.assert_called_with(ExampleConfig.cache_key_name(), current, 300)

    def test_no_config_full_cache(self, mock_cache):
        current = ExampleConfig.current()
        self.assertEquals(current, mock_cache.get.return_value)

    def test_config_ordering(self, mock_cache):
        mock_cache.get.return_value = None

        with freeze_time('2012-01-01'):
            first = ExampleConfig(changed_by=self.user)
            first.string_field = 'first'
            first.save()

        second = ExampleConfig(changed_by=self.user)
        second.string_field = 'second'
        second.save()

        self.assertEquals(ExampleConfig.current().string_field, 'second')

    def test_cache_set(self, mock_cache):
        mock_cache.get.return_value = None

        first = ExampleConfig(changed_by=self.user)
        first.string_field = 'first'
        first.save()

        ExampleConfig.current()

        mock_cache.set.assert_called_with(ExampleConfig.cache_key_name(), first, 300)

########NEW FILE########
__FILENAME__ = middleware
from django.http import (HttpResponse, HttpResponseNotModified,
    HttpResponseForbidden)
from student.models import CourseEnrollment

from xmodule.contentstore.django import contentstore
from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
from xmodule.modulestore import InvalidLocationError, InvalidKeyError
from cache_toolbox.core import get_cached_content, set_cached_content
from xmodule.exceptions import NotFoundError

# TODO: Soon as we have a reasonable way to serialize/deserialize AssetKeys, we need
# to change this file so instead of using course_id_partial, we're just using asset keys

class StaticContentServer(object):
    def process_request(self, request):
        # look to see if the request is prefixed with 'c4x' tag
        if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
            try:
                loc = StaticContent.get_location_from_path(request.path)
            except (InvalidLocationError, InvalidKeyError):
                # return a 'Bad Request' to browser as we have a malformed Location
                response = HttpResponse()
                response.status_code = 400
                return response

            # first look in our cache so we don't have to round-trip to the DB
            content = get_cached_content(loc)
            if content is None:
                # nope, not in cache, let's fetch from DB
                try:
                    content = contentstore().find(loc, as_stream=True)
                except NotFoundError:
                    response = HttpResponse()
                    response.status_code = 404
                    return response

                # since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
                # this is because I haven't been able to find a means to stream data out of memcached
                if content.length is not None:
                    if content.length < 1048576:
                        # since we've queried as a stream, let's read in the stream into memory to set in cache
                        content = content.copy_to_in_mem()
                        set_cached_content(content)
            else:
                # NOP here, but we may wish to add a "cache-hit" counter in the future
                pass

            # Check that user has access to content
            if getattr(content, "locked", False):
                if not hasattr(request, "user") or not request.user.is_authenticated():
                    return HttpResponseForbidden('Unauthorized')
                if not request.user.is_staff and not CourseEnrollment.is_enrolled_by_partial(
                        request.user, loc.course_key
                ):
                    return HttpResponseForbidden('Unauthorized')

            # convert over the DB persistent last modified timestamp to a HTTP compatible
            # timestamp, so we can simply compare the strings
            last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")

            # see if the client has cached this content, if so then compare the
            # timestamps, if they are the same then just return a 304 (Not Modified)
            if 'HTTP_IF_MODIFIED_SINCE' in request.META:
                if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
                if if_modified_since == last_modified_at_str:
                    return HttpResponseNotModified()

            response = HttpResponse(content.stream_data(), content_type=content.content_type)
            response['Last-Modified'] = last_modified_at_str

            return response

########NEW FILE########
__FILENAME__ = test
"""
Tests for StaticContentServer
"""
import copy
import logging
from uuid import uuid4
from path import path
from pymongo import MongoClient

from django.contrib.auth.models import User
from django.conf import settings
from django.test.client import Client
from django.test.utils import override_settings

from student.models import CourseEnrollment

from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.django_utils import (studio_store_config,
    ModuleStoreTestCase)
from xmodule.modulestore.xml_importer import import_from_xml

log = logging.getLogger(__name__)

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex

TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
    """
    Tests that use the toy course.
    """

    def setUp(self):
        """
        Create user and login.
        """

        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')

        self.client = Client()
        self.contentstore = contentstore()

        self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        import_from_xml(modulestore('direct'), 'common/test/data/', ['toy'],
                static_content_store=self.contentstore, verbose=True)

        # A locked asset
        self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt')
        self.url_locked = self.locked_asset.to_deprecated_string()

        # An unlocked asset
        self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt')
        self.url_unlocked = self.unlocked_asset.to_deprecated_string()

        self.contentstore.set_attr(self.locked_asset, 'locked', True)

        # Create user
        self.usr = 'testuser'
        self.pwd = 'foo'
        email = 'test+courses@edx.org'
        self.user = User.objects.create_user(self.usr, email, self.pwd)
        self.user.is_active = True
        self.user.save()

        # Create staff user
        self.staff_usr = 'stafftestuser'
        self.staff_pwd = 'foo'
        staff_email = 'stafftest+courses@edx.org'
        self.staff_user = User.objects.create_user(self.staff_usr, staff_email,
                self.staff_pwd)
        self.staff_user.is_active = True
        self.staff_user.is_staff = True
        self.staff_user.save()

    def tearDown(self):

        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_unlocked_asset(self):
        """
        Test that unlocked assets are being served.
        """
        self.client.logout()
        resp = self.client.get(self.url_unlocked)
        self.assertEqual(resp.status_code, 200) # pylint: disable=E1103

    def test_locked_asset_not_logged_in(self):
        """
        Test that locked assets behave appropriately in case the user is not
        logged in.
        """
        self.client.logout()
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 403) # pylint: disable=E1103

    def test_locked_asset_not_registered(self):
        """
        Test that locked assets behave appropriately in case user is logged in
        in but not registered for the course.
        """
        self.client.login(username=self.usr, password=self.pwd)
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 403) # pylint: disable=E1103

    def test_locked_asset_registered(self):
        """
        Test that locked assets behave appropriately in case user is logged in
        and registered for the course.
        """
        CourseEnrollment.enroll(self.user, self.course_key)
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        self.client.login(username=self.usr, password=self.pwd)
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 200) # pylint: disable=E1103

    def test_locked_asset_staff(self):
        """
        Test that locked assets behave appropriately in case user is staff.
        """
        self.client.login(username=self.staff_usr, password=self.staff_pwd)
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 200) # pylint: disable=E1103


########NEW FILE########
__FILENAME__ = cohorts
"""
This file contains the logic for cohort groups, as exposed internally to the
forums, and to the cohort admin views.
"""

from django.http import Http404
import logging
import random

from courseware import courses
from student.models import get_user_by_username_or_email
from .models import CourseUserGroup

log = logging.getLogger(__name__)


# tl;dr: global state is bad.  capa reseeds random every time a problem is loaded.  Even
# if and when that's fixed, it's a good idea to have a local generator to avoid any other
# code that messes with the global random module.
_local_random = None

def local_random():
    """
    Get the local random number generator.  In a function so that we don't run
    random.Random() at import time.
    """
    # ironic, isn't it?
    global _local_random

    if _local_random is None:
        _local_random = random.Random()

    return _local_random


def is_course_cohorted(course_key):
    """
    Given a course key, return a boolean for whether or not the course is
    cohorted.

    Raises:
       Http404 if the course doesn't exist.
    """
    return courses.get_course_by_id(course_key).is_cohorted


def get_cohort_id(user, course_key):
    """
    Given a course key and a user, return the id of the cohort that user is
    assigned to in that course.  If they don't have a cohort, return None.
    """
    cohort = get_cohort(user, course_key)
    return None if cohort is None else cohort.id


def is_commentable_cohorted(course_key, commentable_id):
    """
    Args:
        course_key: CourseKey
        commentable_id: string

    Returns:
        Bool: is this commentable cohorted?

    Raises:
        Http404 if the course doesn't exist.
    """
    course = courses.get_course_by_id(course_key)

    if not course.is_cohorted:
        # this is the easy case :)
        ans = False
    elif commentable_id in course.top_level_discussion_topic_ids:
        # top level discussions have to be manually configured as cohorted
        # (default is not)
        ans = commentable_id in course.cohorted_discussions
    else:
        # inline discussions are cohorted by default
        ans = True

    log.debug(u"is_commentable_cohorted({0}, {1}) = {2}".format(
        course_key, commentable_id, ans
    ))
    return ans


def get_cohorted_commentables(course_key):
    """
    Given a course_key return a list of strings representing cohorted commentables
    """

    course = courses.get_course_by_id(course_key)

    if not course.is_cohorted:
        # this is the easy case :)
        ans = []
    else:
        ans = course.cohorted_discussions

    return ans


def get_cohort(user, course_key):
    """
    Given a django User and a CourseKey, return the user's cohort in that
    cohort.

    Arguments:
        user: a Django User object.
        course_key: CourseKey

    Returns:
        A CourseUserGroup object if the course is cohorted and the User has a
        cohort, else None.

    Raises:
       ValueError if the CourseKey doesn't exist.
    """
    # First check whether the course is cohorted (users shouldn't be in a cohort
    # in non-cohorted courses, but settings can change after course starts)
    try:
        course = courses.get_course_by_id(course_key)
    except Http404:
        raise ValueError("Invalid course_key")

    if not course.is_cohorted:
        return None

    try:
        return CourseUserGroup.objects.get(course_id=course_key,
                                            group_type=CourseUserGroup.COHORT,
                                            users__id=user.id)
    except CourseUserGroup.DoesNotExist:
        # Didn't find the group.  We'll go on to create one if needed.
        pass

    if not course.auto_cohort:
        return None

    choices = course.auto_cohort_groups
    n = len(choices)
    if n == 0:
        # Nowhere to put user
        log.warning("Course %s is auto-cohorted, but there are no"
                    " auto_cohort_groups specified",
                    course_key)
        return None

    # Put user in a random group, creating it if needed
    group_name = local_random().choice(choices)

    group, created = CourseUserGroup.objects.get_or_create(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        name=group_name
    )

    user.course_groups.add(group)
    return group


def get_course_cohorts(course_key):
    """
    Get a list of all the cohorts in the given course.

    Arguments:
        course_key: CourseKey

    Returns:
        A list of CourseUserGroup objects.  Empty if there are no cohorts. Does
        not check whether the course is cohorted.
    """
    return list(CourseUserGroup.objects.filter(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT
    ))

### Helpers for cohort management views


def get_cohort_by_name(course_key, name):
    """
    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
    it isn't present.
    """
    return CourseUserGroup.objects.get(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        name=name
    )


def get_cohort_by_id(course_key, cohort_id):
    """
    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
    it isn't present.  Uses the course_key for extra validation...
    """
    return CourseUserGroup.objects.get(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        id=cohort_id
    )


def add_cohort(course_key, name):
    """
    Add a cohort to a course.  Raises ValueError if a cohort of the same name already
    exists.
    """
    log.debug("Adding cohort %s to %s", name, course_key)
    if CourseUserGroup.objects.filter(course_id=course_key,
                                      group_type=CourseUserGroup.COHORT,
                                      name=name).exists():
        raise ValueError("Can't create two cohorts with the same name")

    return CourseUserGroup.objects.create(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        name=name
    )


class CohortConflict(Exception):
    """
    Raised when user to be added is already in another cohort in same course.
    """
    pass


def add_user_to_cohort(cohort, username_or_email):
    """
    Look up the given user, and if successful, add them to the specified cohort.

    Arguments:
        cohort: CourseUserGroup
        username_or_email: string.  Treated as email if has '@'

    Returns:
        Tuple of User object and string (or None) indicating previous cohort

    Raises:
        User.DoesNotExist if can't find user.

        ValueError if user already present in this cohort.
    """
    user = get_user_by_username_or_email(username_or_email)
    previous_cohort = None

    course_cohorts = CourseUserGroup.objects.filter(
        course_id=cohort.course_id,
        users__id=user.id,
        group_type=CourseUserGroup.COHORT
    )
    if course_cohorts.exists():
        if course_cohorts[0] == cohort:
            raise ValueError("User {0} already present in cohort {1}".format(
                user.username,
                cohort.name))
        else:
            previous_cohort = course_cohorts[0].name
            course_cohorts[0].users.remove(user)

    cohort.users.add(user)
    return (user, previous_cohort)


def get_course_cohort_names(course_key):
    """
    Return a list of the cohort names in a course.
    """
    return [c.name for c in get_course_cohorts(course_key)]


def delete_empty_cohort(course_key, name):
    """
    Remove an empty cohort.  Raise ValueError if cohort is not empty.
    """
    cohort = get_cohort_by_name(course_key, name)
    if cohort.users.exists():
        raise ValueError(
            "Can't delete non-empty cohort {0} in course {1}".format(
                name, course_key))

    cohort.delete()

########NEW FILE########
__FILENAME__ = models
import logging

from django.contrib.auth.models import User
from django.db import models
from xmodule_django.models import CourseKeyField

log = logging.getLogger(__name__)


class CourseUserGroup(models.Model):
    """
    This model represents groups of users in a course.  Groups may have different types,
    which may be treated specially.  For example, a user can be in at most one cohort per
    course, and cohorts are used to split up the forums by group.
    """
    class Meta:
        unique_together = (('name', 'course_id'), )

    name = models.CharField(max_length=255,
                            help_text=("What is the name of this group?  "
                                       "Must be unique within a course."))
    users = models.ManyToManyField(User, db_index=True, related_name='course_groups',
                                   help_text="Who is in this group?")

    # Note: groups associated with particular runs of a course.  E.g. Fall 2012 and Spring
    # 2013 versions of 6.00x will have separate groups.
    course_id = CourseKeyField(max_length=255, db_index=True,
                                 help_text="Which course is this group associated with?")

    # For now, only have group type 'cohort', but adding a type field to support
    # things like 'question_discussion', 'friends', 'off-line-class', etc
    COHORT = 'cohort'
    GROUP_TYPE_CHOICES = ((COHORT, 'Cohort'),)
    group_type = models.CharField(max_length=20, choices=GROUP_TYPE_CHOICES)

########NEW FILE########
__FILENAME__ = test_cohorts
import django.test
from django.contrib.auth.models import User
from django.conf import settings

from django.test.utils import override_settings

from course_groups.models import CourseUserGroup
from course_groups.cohorts import (get_cohort, get_course_cohorts,
                                   is_commentable_cohorted, get_cohort_by_name)

from xmodule.modulestore.django import modulestore, clear_existing_modulestores
from xmodule.modulestore.locations import SlashSeparatedCourseKey

from xmodule.modulestore.tests.django_utils import mixed_store_config

# NOTE: running this with the lms.envs.test config works without
# manually overriding the modulestore.  However, running with
# cms.envs.test doesn't.

TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
TEST_MAPPING = {'edX/toy/2012_Fall': 'xml'}
TEST_DATA_MIXED_MODULESTORE = mixed_store_config(TEST_DATA_DIR, TEST_MAPPING)


@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class TestCohorts(django.test.TestCase):

    @staticmethod
    def topic_name_to_id(course, name):
        """
        Given a discussion topic name, return an id for that name (includes
        course and url_name).
        """
        return "{course}_{run}_{name}".format(course=course.location.course,
                                              run=course.url_name,
                                              name=name)

    @staticmethod
    def config_course_cohorts(course, discussions,
                              cohorted,
                              cohorted_discussions=None,
                              auto_cohort=None,
                              auto_cohort_groups=None):
        """
        Given a course with no discussion set up, add the discussions and set
        the cohort config appropriately.

        Arguments:
            course: CourseDescriptor
            discussions: list of topic names strings.  Picks ids and sort_keys
                automatically.
            cohorted: bool.
            cohorted_discussions: optional list of topic names.  If specified,
                converts them to use the same ids as topic names.
            auto_cohort: optional bool.
            auto_cohort_groups: optional list of strings
                      (names of groups to put students into).

        Returns:
            Nothing -- modifies course in place.
        """
        def to_id(name):
            return TestCohorts.topic_name_to_id(course, name)

        topics = dict((name, {"sort_key": "A",
                              "id": to_id(name)})
                      for name in discussions)

        course.discussion_topics = topics

        d = {"cohorted": cohorted}
        if cohorted_discussions is not None:
            d["cohorted_discussions"] = [to_id(name)
                                         for name in cohorted_discussions]

        if auto_cohort is not None:
            d["auto_cohort"] = auto_cohort
        if auto_cohort_groups is not None:
            d["auto_cohort_groups"] = auto_cohort_groups

        course.cohort_config = d

    def setUp(self):
        """
        Make sure that course is reloaded every time--clear out the modulestore.
        """
        clear_existing_modulestores()
        self.toy_course_key = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")

    def test_get_cohort(self):
        """
        Make sure get_cohort() does the right thing when the course is cohorted
        """
        course = modulestore().get_course(self.toy_course_key)
        self.assertEqual(course.id, self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        user = User.objects.create(username="test", email="a@b.com")
        other_user = User.objects.create(username="test2", email="a2@b.com")

        self.assertIsNone(get_cohort(user, course.id), "No cohort created yet")

        cohort = CourseUserGroup.objects.create(name="TestCohort",
                                                course_id=course.id,
                                                group_type=CourseUserGroup.COHORT)

        cohort.users.add(user)

        self.assertIsNone(get_cohort(user, course.id),
                          "Course isn't cohorted, so shouldn't have a cohort")

        # Make the course cohorted...
        self.config_course_cohorts(course, [], cohorted=True)

        self.assertEquals(get_cohort(user, course.id).id, cohort.id,
                          "Should find the right cohort")

        self.assertEquals(get_cohort(other_user, course.id), None,
                          "other_user shouldn't have a cohort")

    def test_auto_cohorting(self):
        """
        Make sure get_cohort() does the right thing when the course is auto_cohorted
        """
        course = modulestore().get_course(self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        user1 = User.objects.create(username="test", email="a@b.com")
        user2 = User.objects.create(username="test2", email="a2@b.com")
        user3 = User.objects.create(username="test3", email="a3@b.com")

        cohort = CourseUserGroup.objects.create(name="TestCohort",
                                                course_id=course.id,
                                                group_type=CourseUserGroup.COHORT)

        # user1 manually added to a cohort
        cohort.users.add(user1)

        # Make the course auto cohorted...
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=["AutoGroup"])

        self.assertEquals(get_cohort(user1, course.id).id, cohort.id,
                          "user1 should stay put")

        self.assertEquals(get_cohort(user2, course.id).name, "AutoGroup",
                          "user2 should be auto-cohorted")

        # Now make the group list empty
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=[])

        self.assertEquals(get_cohort(user3, course.id), None,
                          "No groups->no auto-cohorting")

        # Now make it different
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=["OtherGroup"])

        self.assertEquals(get_cohort(user3, course.id).name, "OtherGroup",
                          "New list->new group")
        self.assertEquals(get_cohort(user2, course.id).name, "AutoGroup",
                          "user2 should still be in originally placed cohort")

    def test_auto_cohorting_randomization(self):
        """
        Make sure get_cohort() randomizes properly.
        """
        course = modulestore().get_course(self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        groups = ["group_{0}".format(n) for n in range(5)]
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=groups)

        # Assign 100 users to cohorts
        for i in range(100):
            user = User.objects.create(username="test_{0}".format(i),
                                       email="a@b{0}.com".format(i))
            get_cohort(user, course.id)

        # Now make sure that the assignment was at least vaguely random:
        # each cohort should have at least 1, and fewer than 50 students.
        # (with 5 groups, probability of 0 users in any group is about
        # .8**100= 2.0e-10)
        for cohort_name in groups:
            cohort = get_cohort_by_name(course.id, cohort_name)
            num_users = cohort.users.count()
            self.assertGreater(num_users, 1)
            self.assertLess(num_users, 50)

    def test_get_course_cohorts(self):
        course1_key = SlashSeparatedCourseKey('a', 'b', 'c')
        course2_key = SlashSeparatedCourseKey('e', 'f', 'g')

        # add some cohorts to course 1
        cohort = CourseUserGroup.objects.create(name="TestCohort",
                                                course_id=course1_key,
                                                group_type=CourseUserGroup.COHORT)

        cohort = CourseUserGroup.objects.create(name="TestCohort2",
                                                course_id=course1_key,
                                                group_type=CourseUserGroup.COHORT)

        # second course should have no cohorts
        self.assertEqual(get_course_cohorts(course2_key), [])

        cohorts = sorted([c.name for c in get_course_cohorts(course1_key)])
        self.assertEqual(cohorts, ['TestCohort', 'TestCohort2'])

    def test_is_commentable_cohorted(self):
        course = modulestore().get_course(self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        def to_id(name):
            return self.topic_name_to_id(course, name)

        # no topics
        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course doesn't even have a 'General' topic")

        # not cohorted
        self.config_course_cohorts(course, ["General", "Feedback"],
                                   cohorted=False)

        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course isn't cohorted")

        # cohorted, but top level topics aren't
        self.config_course_cohorts(course, ["General", "Feedback"],
                                   cohorted=True)

        self.assertTrue(course.is_cohorted)
        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course is cohorted, but 'General' isn't.")

        self.assertTrue(
            is_commentable_cohorted(course.id, to_id("random")),
            "Non-top-level discussion is always cohorted in cohorted courses.")

        # cohorted, including "Feedback" top-level topics aren't
        self.config_course_cohorts(course, ["General", "Feedback"],
                                   cohorted=True,
                                   cohorted_discussions=["Feedback"])

        self.assertTrue(course.is_cohorted)
        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course is cohorted, but 'General' isn't.")

        self.assertTrue(
            is_commentable_cohorted(course.id, to_id("Feedback")),
            "Feedback was listed as cohorted.  Should be.")

########NEW FILE########
__FILENAME__ = test_views
import json

from django.test.client import RequestFactory
from django.test.utils import override_settings
from factory import post_generation, Sequence
from factory.django import DjangoModelFactory

from course_groups.models import CourseUserGroup
from course_groups.views import add_users_to_cohort
from courseware.tests.tests import TEST_DATA_MIXED_MODULESTORE
from student.tests.factories import UserFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory


class CohortFactory(DjangoModelFactory):
    FACTORY_FOR = CourseUserGroup

    name = Sequence("cohort{}".format)
    course_id = "dummy_id"
    group_type = CourseUserGroup.COHORT

    @post_generation
    def users(self, create, extracted, **kwargs):  # pylint: disable=W0613
        self.users.add(*extracted)


@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class AddUsersToCohortTestCase(ModuleStoreTestCase):
    def setUp(self):
        self.course = CourseFactory.create()
        self.staff_user = UserFactory.create(is_staff=True)
        self.cohort1_users = [UserFactory.create() for _ in range(3)]
        self.cohort2_users = [UserFactory.create() for _ in range(2)]
        self.cohort3_users = [UserFactory.create() for _ in range(2)]
        self.cohortless_users = [UserFactory.create() for _ in range(3)]
        self.cohort1 = CohortFactory.create(course_id=self.course.id, users=self.cohort1_users)
        self.cohort2 = CohortFactory.create(course_id=self.course.id, users=self.cohort2_users)
        self.cohort3 = CohortFactory.create(course_id=self.course.id, users=self.cohort3_users)

    def check_request(
            self,
            users_string,
            expected_added=None,
            expected_changed=None,
            expected_present=None,
            expected_unknown=None
    ):
        """
        Check that add_users_to_cohort returns the expected result and has the
        expected side effects. The given users will be added to cohort1.

        users_string is the string input entered by the client

        expected_added is a list of users

        expected_changed is a list of (user, previous_cohort) tuples

        expected_present is a list of (user, email/username) tuples where
        email/username corresponds to the input

        expected_unknown is a list of strings corresponding to the input
        """
        expected_added = expected_added or []
        expected_changed = expected_changed or []
        expected_present = expected_present or []
        expected_unknown = expected_unknown or []
        request = RequestFactory().post("dummy_url", {"users": users_string})
        request.user = self.staff_user
        response = add_users_to_cohort(request, self.course.id.to_deprecated_string(), self.cohort1.id)
        self.assertEqual(response.status_code, 200)
        result = json.loads(response.content)
        self.assertEqual(result.get("success"), True)
        self.assertItemsEqual(
            result.get("added"),
            [
                {"username": user.username, "name": user.profile.name, "email": user.email}
                for user in expected_added
            ]
        )
        self.assertItemsEqual(
            result.get("changed"),
            [
                {
                    "username": user.username,
                    "name": user.profile.name,
                    "email": user.email,
                    "previous_cohort": previous_cohort
                }
                for (user, previous_cohort) in expected_changed
            ]
        )
        self.assertItemsEqual(
            result.get("present"),
            [username_or_email for (_, username_or_email) in expected_present]
        )
        self.assertItemsEqual(result.get("unknown"), expected_unknown)
        for user in expected_added + [user for (user, _) in expected_changed + expected_present]:
            self.assertEqual(
                CourseUserGroup.objects.get(
                    course_id=self.course.id,
                    group_type=CourseUserGroup.COHORT,
                    users__id=user.id
                ),
                self.cohort1
            )

    def test_empty(self):
        self.check_request("")

    def test_only_added(self):
        self.check_request(
            ",".join([user.username for user in self.cohortless_users]),
            expected_added=self.cohortless_users
        )

    def test_only_changed(self):
        self.check_request(
            ",".join([user.username for user in self.cohort2_users + self.cohort3_users]),
            expected_changed=(
                [(user, self.cohort2.name) for user in self.cohort2_users] +
                [(user, self.cohort3.name) for user in self.cohort3_users]
            )
        )

    def test_only_present(self):
        usernames = [user.username for user in self.cohort1_users]
        self.check_request(
            ",".join(usernames),
            expected_present=[(user, user.username) for user in self.cohort1_users]
        )

    def test_only_unknown(self):
        usernames = ["unknown_user{}".format(i) for i in range(3)]
        self.check_request(
            ",".join(usernames),
            expected_unknown=usernames
        )

    def test_all(self):
        unknowns = ["unknown_user{}".format(i) for i in range(3)]
        self.check_request(
            ",".join(
                unknowns +
                [
                    user.username
                    for user in self.cohortless_users + self.cohort1_users + self.cohort2_users + self.cohort3_users
                ]
            ),
            self.cohortless_users,
            (
                [(user, self.cohort2.name) for user in self.cohort2_users] +
                [(user, self.cohort3.name) for user in self.cohort3_users]
            ),
            [(user, user.username) for user in self.cohort1_users],
            unknowns
        )

    def test_emails(self):
        unknown = "unknown_user@example.com"
        self.check_request(
            ",".join([
                self.cohort1_users[0].email,
                self.cohort2_users[0].email,
                self.cohortless_users[0].email,
                unknown
            ]),
            [self.cohortless_users[0]],
            [(self.cohort2_users[0], self.cohort2.name)],
            [(self.cohort1_users[0], self.cohort1_users[0].email)],
            [unknown]
        )

    def test_delimiters(self):
        unknown = "unknown_user"
        self.check_request(
            " {} {}\t{}, \r\n{}".format(
                unknown,
                self.cohort1_users[0].username,
                self.cohort2_users[0].username,
                self.cohortless_users[0].username
            ),
            [self.cohortless_users[0]],
            [(self.cohort2_users[0], self.cohort2.name)],
            [(self.cohort1_users[0], self.cohort1_users[0].username)],
            [unknown]
        )

########NEW FILE########
__FILENAME__ = views
from django_future.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_POST
from django.contrib.auth.models import User
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.http import HttpResponse
import json
import logging
import re

from xmodule.modulestore.locations import SlashSeparatedCourseKey
from courseware.courses import get_course_with_access
from edxmako.shortcuts import render_to_response

from . import cohorts


log = logging.getLogger(__name__)


def json_http_response(data):
    """
    Return an HttpResponse with the data json-serialized and the right content
    type header.
    """
    return HttpResponse(json.dumps(data), content_type="application/json")


def split_by_comma_and_whitespace(s):
    """
    Split a string both by commas and whitespice.  Returns a list.
    """
    return re.split(r'[\s,]+', s)


@ensure_csrf_cookie
def list_cohorts(request, course_key):
    """
    Return json dump of dict:

    {'success': True,
     'cohorts': [{'name': name, 'id': id}, ...]}
    """

    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)

    get_course_with_access(request.user, 'staff', course_key)

    all_cohorts = [{'name': c.name, 'id': c.id}
                   for c in cohorts.get_course_cohorts(course_key)]

    return json_http_response({'success': True,
                               'cohorts': all_cohorts})


@ensure_csrf_cookie
@require_POST
def add_cohort(request, course_key):
    """
    Return json of dict:
    {'success': True,
     'cohort': {'id': id,
                'name': name}}

                or

    {'success': False,
     'msg': error_msg} if there's an error
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)

    get_course_with_access(request.user, 'staff', course_key)

    name = request.POST.get("name")
    if not name:
        return json_http_response({'success': False,
                                'msg': "No name specified"})

    try:
        cohort = cohorts.add_cohort(course_key, name)
    except ValueError as err:
        return json_http_response({'success': False,
                                'msg': str(err)})

    return json_http_response({'success': 'True',
                            'cohort': {
                                'id': cohort.id,
                                'name': cohort.name
                                }})


@ensure_csrf_cookie
def users_in_cohort(request, course_key, cohort_id):
    """
    Return users in the cohort.  Show up to 100 per page, and page
    using the 'page' GET attribute in the call.  Format:

    Returns:
        Json dump of dictionary in the following format:
        {'success': True,
         'page': page,
         'num_pages': paginator.num_pages,
         'users': [{'username': ..., 'email': ..., 'name': ...}]
    }
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)

    get_course_with_access(request.user, 'staff', course_key)

    # this will error if called with a non-int cohort_id.  That's ok--it
    # shoudn't happen for valid clients.
    cohort = cohorts.get_cohort_by_id(course_key, int(cohort_id))

    paginator = Paginator(cohort.users.all(), 100)
    page = request.GET.get('page')
    try:
        users = paginator.page(page)
    except PageNotAnInteger:
        # return the first page
        page = 1
        users = paginator.page(page)
    except EmptyPage:
        # Page is out of range.  Return last page
        page = paginator.num_pages
        contacts = paginator.page(page)

    user_info = [{'username': u.username,
                  'email': u.email,
                  'name': '{0} {1}'.format(u.first_name, u.last_name)}
                 for u in users]

    return json_http_response({'success': True,
                               'page': page,
                               'num_pages': paginator.num_pages,
                               'users': user_info})


@ensure_csrf_cookie
@require_POST
def add_users_to_cohort(request, course_key, cohort_id):
    """
    Return json dict of:

    {'success': True,
     'added': [{'username': ...,
                'name': ...,
                'email': ...}, ...],
     'changed': [{'username': ...,
                  'name': ...,
                  'email': ...,
                  'previous_cohort': ...}, ...],
     'present': [str1, str2, ...],    # already there
     'unknown': [str1, str2, ...]}
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)
    get_course_with_access(request.user, 'staff', course_key)

    cohort = cohorts.get_cohort_by_id(course_key, cohort_id)

    users = request.POST.get('users', '')
    added = []
    changed = []
    present = []
    unknown = []
    for username_or_email in split_by_comma_and_whitespace(users):
        if not username_or_email:
            continue

        try:
            (user, previous_cohort) = cohorts.add_user_to_cohort(cohort, username_or_email)
            info = {
                'username': user.username,
                'name': user.profile.name,
                'email': user.email,
            }
            if previous_cohort:
                info['previous_cohort'] = previous_cohort
                changed.append(info)
            else:
                added.append(info)
        except ValueError:
            present.append(username_or_email)
        except User.DoesNotExist:
            unknown.append(username_or_email)

    return json_http_response({'success': True,
                               'added': added,
                               'changed': changed,
                               'present': present,
                               'unknown': unknown})


@ensure_csrf_cookie
@require_POST
def remove_user_from_cohort(request, course_key, cohort_id):
    """
    Expects 'username': username in POST data.

    Return json dict of:

    {'success': True} or
    {'success': False,
     'msg': error_msg}
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)
    get_course_with_access(request.user, 'staff', course_key)

    username = request.POST.get('username')
    if username is None:
        return json_http_response({'success': False,
                                   'msg': 'No username specified'})

    cohort = cohorts.get_cohort_by_id(course_key, cohort_id)
    try:
        user = User.objects.get(username=username)
        cohort.users.remove(user)
        return json_http_response({'success': True})
    except User.DoesNotExist:
        log.debug('no user')
        return json_http_response({'success': False,
                                   'msg': "No user '{0}'".format(username)})


def debug_cohort_mgmt(request, course_key):
    """
    Debugging view for dev.
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)
    # add staff check to make sure it's safe if it's accidentally deployed.
    get_course_with_access(request.user, 'staff', course_key)

    context = {'cohorts_ajax_url': reverse(
        'cohorts',
        kwargs={'course_key': course_key.to_deprecated_string()}
    )}
    return render_to_response('/course_groups/debug.html', context)

########NEW FILE########
__FILENAME__ = admin
from ratelimitbackend import admin
from course_modes.models import CourseMode
from django import forms

from opaque_keys import InvalidKeyError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class CourseModeForm(forms.ModelForm):

    class Meta:
        model = CourseMode

    def clean_course_id(self):
        course_id = self.cleaned_data['course_id']
        try:
            course_key = CourseKey.from_string(course_id)
        except InvalidKeyError:
            try:
                course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
            except InvalidKeyError:
                raise forms.ValidationError("Cannot make a valid CourseKey from id {}!".format(course_id))

        if not modulestore().has_course(course_key):
            raise forms.ValidationError("Cannot find course with id {} in the modulestore".format(course_id))

        return course_key


class CourseModeAdmin(admin.ModelAdmin):
    form = CourseModeForm


admin.site.register(CourseMode, CourseModeAdmin)

########NEW FILE########
__FILENAME__ = 0001_initial
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'CourseMode'
        db.create_table('course_modes_coursemode', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
            ('mode_slug', self.gf('django.db.models.fields.CharField')(max_length=100)),
            ('mode_display_name', self.gf('django.db.models.fields.CharField')(max_length=255)),
            ('min_price', self.gf('django.db.models.fields.IntegerField')(default=0)),
            ('suggested_prices', self.gf('django.db.models.fields.CommaSeparatedIntegerField')(default='', max_length=255, blank=True)),
        ))
        db.send_create_signal('course_modes', ['CourseMode'])


    def backwards(self, orm):
        # Deleting model 'CourseMode'
        db.delete_table('course_modes_coursemode')


    models = {
        'course_modes.coursemode': {
            'Meta': {'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0002_auto__add_field_coursemode_currency
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'CourseMode.currency'
        db.add_column('course_modes_coursemode', 'currency',
                      self.gf('django.db.models.fields.CharField')(default='usd', max_length=8),
                      keep_default=False)


    def backwards(self, orm):
        # Deleting field 'CourseMode.currency'
        db.delete_column('course_modes_coursemode', 'currency')


    models = {
        'course_modes.coursemode': {
            'Meta': {'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0003_auto__add_unique_coursemode_course_id_currency_mode_slug
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding unique constraint on 'CourseMode', fields ['course_id', 'currency', 'mode_slug']
        db.create_unique('course_modes_coursemode', ['course_id', 'currency', 'mode_slug'])


    def backwards(self, orm):
        # Removing unique constraint on 'CourseMode', fields ['course_id', 'currency', 'mode_slug']
        db.delete_unique('course_modes_coursemode', ['course_id', 'currency', 'mode_slug'])


    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0004_auto__add_field_coursemode_expiration_date
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'CourseMode.expiration_date'
        db.add_column('course_modes_coursemode', 'expiration_date',
                      self.gf('django.db.models.fields.DateField')(default=None, null=True),
                      keep_default=False)


    def backwards(self, orm):
        # Deleting field 'CourseMode.expiration_date'
        db.delete_column('course_modes_coursemode', 'expiration_date')


    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'expiration_date': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0005_auto__add_field_coursemode_expiration_datetime
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'CourseMode.expiration_datetime'
        db.add_column('course_modes_coursemode', 'expiration_datetime',
                      self.gf('django.db.models.fields.DateTimeField')(default=None, null=True, blank=True),
                      keep_default=False)


    def backwards(self, orm):
        # Deleting field 'CourseMode.expiration_datetime'
        db.delete_column('course_modes_coursemode', 'expiration_datetime')


    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'expiration_date': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'expiration_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0006_expiration_date_to_datetime
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        from datetime import datetime
        for course_mode in orm.CourseMode.objects.all():
            if course_mode.expiration_date is None:
                course_mode.expiration_datetime = None
                course_mode.save()
            else:
                course_mode.expiration_datetime = datetime.combine(course_mode.expiration_date, datetime.min.time())
                course_mode.save()

    def backwards(self, orm):
        for course_mode in orm.CourseMode.objects.all():
            course_mode.expiration_datetime = None
            course_mode.save()

    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'expiration_date': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'expiration_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
    symmetrical = True

########NEW FILE########
__FILENAME__ = models
"""
Add and create new modes for running courses on this particular LMS
"""
import pytz
from datetime import datetime

from django.db import models
from collections import namedtuple
from django.utils.translation import ugettext as _
from django.db.models import Q

from xmodule_django.models import CourseKeyField

Mode = namedtuple('Mode', ['slug', 'name', 'min_price', 'suggested_prices', 'currency', 'expiration_datetime'])

class CourseMode(models.Model):
    """
    We would like to offer a course in a variety of modes.

    """
    # the course that this mode is attached to
    course_id = CourseKeyField(max_length=255, db_index=True)

    # the reference to this mode that can be used by Enrollments to generate
    # similar behavior for the same slug across courses
    mode_slug = models.CharField(max_length=100)

    # The 'pretty' name that can be translated and displayed
    mode_display_name = models.CharField(max_length=255)

    # minimum price in USD that we would like to charge for this mode of the course
    min_price = models.IntegerField(default=0)

    # the suggested prices for this mode
    suggested_prices = models.CommaSeparatedIntegerField(max_length=255, blank=True, default='')

    # the currency these prices are in, using lower case ISO currency codes
    currency = models.CharField(default="usd", max_length=8)

    # turn this mode off after the given expiration date
    expiration_date = models.DateField(default=None, null=True, blank=True)

    expiration_datetime = models.DateTimeField(default=None, null=True, blank=True)

    DEFAULT_MODE = Mode('honor', _('Honor Code Certificate'), 0, '', 'usd', None)
    DEFAULT_MODE_SLUG = 'honor'

    class Meta:
        """ meta attributes of this model """
        unique_together = ('course_id', 'mode_slug', 'currency')

    @classmethod
    def modes_for_course(cls, course_id):
        """
        Returns a list of the non-expired modes for a given course id

        If no modes have been set in the table, returns the default mode
        """
        now = datetime.now(pytz.UTC)
        found_course_modes = cls.objects.filter(Q(course_id=course_id) &
                                                (Q(expiration_datetime__isnull=True) |
                                                Q(expiration_datetime__gte=now)))
        modes = ([Mode(
            mode.mode_slug,
            mode.mode_display_name,
            mode.min_price,
            mode.suggested_prices,
            mode.currency,
            mode.expiration_datetime
        ) for mode in found_course_modes])
        if not modes:
            modes = [cls.DEFAULT_MODE]
        return modes

    @classmethod
    def modes_for_course_dict(cls, course_id):
        """
        Returns the non-expired modes for a particular course as a
        dictionary with the mode slug as the key
        """
        return {mode.slug: mode for mode in cls.modes_for_course(course_id)}

    @classmethod
    def mode_for_course(cls, course_id, mode_slug):
        """
        Returns the mode for the course corresponding to mode_slug.

        Returns only non-expired modes.

        If this particular mode is not set for the course, returns None
        """
        modes = cls.modes_for_course(course_id)

        matched = [m for m in modes if m.slug == mode_slug]
        if matched:
            return matched[0]
        else:
            return None

    @classmethod
    def min_course_price_for_verified_for_currency(cls, course_id, currency):
        """
        Returns the minimum price of the course int he appropriate currency over all the
        course's *verified*, non-expired modes.

        Assuming all verified courses have a minimum price of >0, this value should always
        be >0.

        If no verified mode is found, 0 is returned.
        """
        modes = cls.modes_for_course(course_id)
        for mode in modes:
            if (mode.currency == currency) and (mode.slug == 'verified'):
                return mode.min_price
        return 0

    @classmethod
    def min_course_price_for_currency(cls, course_id, currency):
        """
        Returns the minimum price of the course in the appropriate currency over all the course's
        non-expired modes.
        If there is no mode found, will return the price of DEFAULT_MODE, which is 0
        """
        modes = cls.modes_for_course(course_id)
        return min(mode.min_price for mode in modes if mode.currency == currency)

    def __unicode__(self):
        return u"{} : {}, min={}, prices={}".format(
            self.course_id.to_deprecated_string(), self.mode_slug, self.min_price, self.suggested_prices
        )

########NEW FILE########
__FILENAME__ = factories
from course_modes.models import CourseMode
from factory.django import DjangoModelFactory


# Factories don't have __init__ methods, and are self documenting
# pylint: disable=W0232
class CourseModeFactory(DjangoModelFactory):
    FACTORY_FOR = CourseMode

    course_id = u'MITx/999/Robot_Super_Course'
    mode_slug = 'audit'
    mode_display_name = 'audit course'
    min_price = 0
    currency = 'usd'
    expiration_datetime = None

########NEW FILE########
__FILENAME__ = test_models
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from datetime import datetime, timedelta
import pytz

from xmodule.modulestore.locations import SlashSeparatedCourseKey
from django.test import TestCase
from course_modes.models import CourseMode, Mode


class CourseModeModelTest(TestCase):
    """
    Tests for the CourseMode model
    """

    def setUp(self):
        self.course_key = SlashSeparatedCourseKey('Test', 'TestCourse', 'TestCourseRun')
        CourseMode.objects.all().delete()

    def create_mode(self, mode_slug, mode_name, min_price=0, suggested_prices='', currency='usd'):
        """
        Create a new course mode
        """
        return CourseMode.objects.get_or_create(
            course_id=self.course_key,
            mode_display_name=mode_name,
            mode_slug=mode_slug,
            min_price=min_price,
            suggested_prices=suggested_prices,
            currency=currency,
        )

    def test_modes_for_course_empty(self):
        """
        If we can't find any modes, we should get back the default mode
        """
        # shouldn't be able to find a corresponding course
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([CourseMode.DEFAULT_MODE], modes)

    def test_nodes_for_course_single(self):
        """
        Find the modes for a course with only one mode
        """

        self.create_mode('verified', 'Verified Certificate')
        modes = CourseMode.modes_for_course(self.course_key)
        mode = Mode(u'verified', u'Verified Certificate', 0, '', 'usd', None)
        self.assertEqual([mode], modes)

        modes_dict = CourseMode.modes_for_course_dict(self.course_key)
        self.assertEqual(modes_dict['verified'], mode)
        self.assertEqual(CourseMode.mode_for_course(self.course_key, 'verified'),
                         mode)

    def test_modes_for_course_multiple(self):
        """
        Finding the modes when there's multiple modes
        """
        mode1 = Mode(u'honor', u'Honor Code Certificate', 0, '', 'usd', None)
        mode2 = Mode(u'verified', u'Verified Certificate', 0, '', 'usd', None)
        set_modes = [mode1, mode2]
        for mode in set_modes:
            self.create_mode(mode.slug, mode.name, mode.min_price, mode.suggested_prices)

        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual(modes, set_modes)
        self.assertEqual(mode1, CourseMode.mode_for_course(self.course_key, u'honor'))
        self.assertEqual(mode2, CourseMode.mode_for_course(self.course_key, u'verified'))
        self.assertIsNone(CourseMode.mode_for_course(self.course_key, 'DNE'))

    def test_min_course_price_for_currency(self):
        """
        Get the min course price for a course according to currency
        """
        # no modes, should get 0
        self.assertEqual(0, CourseMode.min_course_price_for_currency(self.course_key, 'usd'))

        # create some modes
        mode1 = Mode(u'honor', u'Honor Code Certificate', 10, '', 'usd', None)
        mode2 = Mode(u'verified', u'Verified Certificate', 20, '', 'usd', None)
        mode3 = Mode(u'honor', u'Honor Code Certificate', 80, '', 'cny', None)
        set_modes = [mode1, mode2, mode3]
        for mode in set_modes:
            self.create_mode(mode.slug, mode.name, mode.min_price, mode.suggested_prices, mode.currency)

        self.assertEqual(10, CourseMode.min_course_price_for_currency(self.course_key, 'usd'))
        self.assertEqual(80, CourseMode.min_course_price_for_currency(self.course_key, 'cny'))

    def test_modes_for_course_expired(self):
        expired_mode, _status = self.create_mode('verified', 'Verified Certificate')
        expired_mode.expiration_datetime = datetime.now(pytz.UTC) + timedelta(days=-1)
        expired_mode.save()
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([CourseMode.DEFAULT_MODE], modes)

        mode1 = Mode(u'honor', u'Honor Code Certificate', 0, '', 'usd', None)
        self.create_mode(mode1.slug, mode1.name, mode1.min_price, mode1.suggested_prices)
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([mode1], modes)

        expiration_datetime = datetime.now(pytz.UTC) + timedelta(days=1)
        expired_mode.expiration_datetime = expiration_datetime
        expired_mode.save()
        expired_mode_value = Mode(u'verified', u'Verified Certificate', 0, '', 'usd', expiration_datetime)
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([expired_mode_value, mode1], modes)

        modes = CourseMode.modes_for_course(SlashSeparatedCourseKey('TestOrg', 'TestCourse', 'TestRun'))
        self.assertEqual([CourseMode.DEFAULT_MODE], modes)

########NEW FILE########
__FILENAME__ = urls
from django.conf.urls import include, patterns, url
from django.views.generic import TemplateView

from course_modes import views

urlpatterns = patterns(
    '',
    url(r'^choose/(?P<course_id>[^/]+/[^/]+/[^/]+)$', views.ChooseModeView.as_view(), name="course_modes_choose"),
)

########NEW FILE########
__FILENAME__ = views
"""
Views for the course_mode module
"""

import decimal
from django.core.urlresolvers import reverse
from django.http import (
    HttpResponseBadRequest,  Http404
)
from django.shortcuts import redirect
from django.views.generic.base import View
from django.utils.translation import ugettext as _
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator

from edxmako.shortcuts import render_to_response

from course_modes.models import CourseMode
from courseware.access import has_access
from student.models import CourseEnrollment
from student.views import course_from_id
from verify_student.models import SoftwareSecurePhotoVerification
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class ChooseModeView(View):
    """
    View used when the user is asked to pick a mode

    When a get request is used, shows the selection page.
    When a post request is used, assumes that it is a form submission
        from the selection page, parses the response, and then sends user
        to the next step in the flow
    """
    @method_decorator(login_required)
    def get(self, request, course_id, error=None):
        """ Displays the course mode choice page """

        course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)

        enrollment_mode = CourseEnrollment.enrollment_mode_for_user(request.user, course_key)
        upgrade = request.GET.get('upgrade', False)
        request.session['attempting_upgrade'] = upgrade

        # verified users do not need to register or upgrade
        if enrollment_mode == 'verified':
            return redirect(reverse('dashboard'))

        # registered users who are not trying to upgrade do not need to re-register
        if enrollment_mode is not None and upgrade is False:
            return redirect(reverse('dashboard'))

        modes = CourseMode.modes_for_course_dict(course_key)
        donation_for_course = request.session.get("donation_for_course", {})
        chosen_price = donation_for_course.get(course_key, None)

        course = course_from_id(course_key)
        context = {
            "course_modes_choose_url": reverse("course_modes_choose", kwargs={'course_id': course_key.to_deprecated_string()}),
            "modes": modes,
            "course_name": course.display_name_with_default,
            "course_org": course.display_org_with_default,
            "course_num": course.display_number_with_default,
            "chosen_price": chosen_price,
            "error": error,
            "upgrade": upgrade,
        }
        if "verified" in modes:
            context["suggested_prices"] = [decimal.Decimal(x) for x in modes["verified"].suggested_prices.split(",")]
            context["currency"] = modes["verified"].currency.upper()
            context["min_price"] = modes["verified"].min_price

        return render_to_response("course_modes/choose.html", context)

    @method_decorator(login_required)
    def post(self, request, course_id):
        """ Takes the form submission from the page and parses it """
        course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
        user = request.user

        # This is a bit redundant with logic in student.views.change_enrollement,
        # but I don't really have the time to refactor it more nicely and test.
        course = course_from_id(course_key)
        if not has_access(user, 'enroll', course):
            error_msg = _("Enrollment is closed")
            return self.get(request, course_id, error=error_msg)

        upgrade = request.GET.get('upgrade', False)

        requested_mode = self.get_requested_mode(request.POST)

        allowed_modes = CourseMode.modes_for_course_dict(course_key)
        if requested_mode not in allowed_modes:
            return HttpResponseBadRequest(_("Enrollment mode not supported"))

        if requested_mode in ("audit", "honor"):
            CourseEnrollment.enroll(user, course_key, requested_mode)
            return redirect('dashboard')

        mode_info = allowed_modes[requested_mode]

        if requested_mode == "verified":
            amount = request.POST.get("contribution") or \
                request.POST.get("contribution-other-amt") or 0

            try:
                # validate the amount passed in and force it into two digits
                amount_value = decimal.Decimal(amount).quantize(decimal.Decimal('.01'), rounding=decimal.ROUND_DOWN)
            except decimal.InvalidOperation:
                error_msg = _("Invalid amount selected.")
                return self.get(request, course_id, error=error_msg)

            # Check for minimum pricing
            if amount_value < mode_info.min_price:
                error_msg = _("No selected price or selected price is too low.")
                return self.get(request, course_id, error=error_msg)

            donation_for_course = request.session.get("donation_for_course", {})
            donation_for_course[course_key] = amount_value
            request.session["donation_for_course"] = donation_for_course
            if SoftwareSecurePhotoVerification.user_has_valid_or_pending(request.user):
                return redirect(
                    reverse('verify_student_verified',
                            kwargs={'course_id': course_key.to_deprecated_string()}) + "?upgrade={}".format(upgrade)
                )

            return redirect(
                reverse('verify_student_show_requirements',
                        kwargs={'course_id': course_key.to_deprecated_string()}) + "?upgrade={}".format(upgrade))

    def get_requested_mode(self, request_dict):
        """
        Given the request object of `user_choice`, return the
        corresponding course mode slug
        """
        if 'audit_mode' in request_dict:
            return 'audit'
        if 'certificate_mode' and request_dict.get("honor-code"):
            return 'honor'
        if 'certificate_mode' in request_dict:
            return 'verified'

########NEW FILE########
__FILENAME__ = admin
"""
Admin site bindings for dark_lang
"""

from django.contrib import admin

from config_models.admin import ConfigurationModelAdmin
from dark_lang.models import DarkLangConfig

admin.site.register(DarkLangConfig, ConfigurationModelAdmin)

########NEW FILE########
__FILENAME__ = middleware
"""
Middleware for dark-launching languages. These languages won't be used
when determining which translation to give a user based on their browser
header, but can be selected by setting the ``preview-lang`` query parameter
to the language code.

Adding the query parameter ``clear-lang`` will reset the language stored
in the user's session.

This middleware must be placed before the LocaleMiddleware, but after
the SessionMiddleware.
"""
from django.conf import settings

from django.utils.translation.trans_real import parse_accept_lang_header

from dark_lang.models import DarkLangConfig


def dark_parse_accept_lang_header(accept):
    '''
    The use of 'zh-cn' for 'Simplified Chinese' and 'zh-tw' for 'Traditional Chinese'
    are now deprecated, as discussed here: https://code.djangoproject.com/ticket/18419.
    The new language codes 'zh-hans' and 'zh-hant' are now used since django 1.7.
    Although majority of browsers still use the old language codes, some new browsers
    such as IE11 in Windows 8.1 start to use the new ones, which makes the current
    chinese translations of edX don't work properly under these browsers.
    This function can keep compatibility between the old and new language codes. If one
    day edX uses django 1.7 or higher, this function can be modified to support the old
    language codes until there are no browsers use them.
    '''
    browser_langs = parse_accept_lang_header(accept)
    django_langs = []
    for lang, priority in browser_langs:
        lang = CHINESE_LANGUAGE_CODE_MAP.get(lang.lower(), lang)
        django_langs.append((lang, priority))

    return django_langs

# If django 1.7 or higher is used, the right-side can be updated with new-style codes.
CHINESE_LANGUAGE_CODE_MAP = {
    # The following are the new-style language codes for chinese language
    'zh-hans': 'zh-CN',     # Chinese (Simplified),
    'zh-hans-cn': 'zh-CN',  # Chinese (Simplified, China)
    'zh-hans-sg': 'zh-CN',  # Chinese (Simplified, Singapore)
    'zh-hant': 'zh-TW',     # Chinese (Traditional)
    'zh-hant-hk': 'zh-TW',  # Chinese (Traditional, Hongkong)
    'zh-hant-mo': 'zh-TW',  # Chinese (Traditional, Macau)
    'zh-hant-tw': 'zh-TW',  # Chinese (Traditional, Taiwan)
    # The following are the old-style language codes that django does not recognize
    'zh-hk': 'zh-TW',       # Chinese (Traditional, Hongkong)
    'zh-mo': 'zh-TW',       # Chinese (Traditional, Macau)
    'zh-sg': 'zh-CN',       # Chinese (Simplified, Singapore)
}


class DarkLangMiddleware(object):
    """
    Middleware for dark-launching languages.

    This is configured by creating ``DarkLangConfig`` rows in the database,
    using the django admin site.
    """

    @property
    def released_langs(self):
        """
        Current list of released languages
        """
        language_options = DarkLangConfig.current().released_languages_list
        if settings.LANGUAGE_CODE not in language_options:
            language_options.append(settings.LANGUAGE_CODE)
        return language_options

    def process_request(self, request):
        """
        Prevent user from requesting un-released languages except by using the preview-lang query string.
        """
        if not DarkLangConfig.current().enabled:
            return

        self._clean_accept_headers(request)
        self._activate_preview_language(request)

    def _is_released(self, lang_code):
        """
        ``True`` iff one of the values in ``self.released_langs`` is a prefix of ``lang_code``.
        """
        return any(lang_code.lower().startswith(released_lang.lower()) for released_lang in self.released_langs)

    def _format_accept_value(self, lang, priority=1.0):
        """
        Formats lang and priority into a valid accept header fragment.
        """
        return "{};q={}".format(lang, priority)

    def _clean_accept_headers(self, request):
        """
        Remove any language that is not either in ``self.released_langs`` or
        a territory of one of those languages.
        """
        accept = request.META.get('HTTP_ACCEPT_LANGUAGE', None)
        if accept is None or accept == '*':
            return

        new_accept = ", ".join(
            self._format_accept_value(lang, priority)
            for lang, priority
            in dark_parse_accept_lang_header(accept)
            if self._is_released(lang)
        )

        request.META['HTTP_ACCEPT_LANGUAGE'] = new_accept

    def _activate_preview_language(self, request):
        """
        If the request has the get parameter ``preview-lang``,
        and that language doesn't appear in ``self.released_langs``,
        then set the session ``django_language`` to that language.
        """
        if 'clear-lang' in request.GET:
            if 'django_language' in request.session:
                del request.session['django_language']

        preview_lang = request.GET.get('preview-lang', None)

        if not preview_lang:
            return

        if preview_lang in self.released_langs:
            return

        request.session['django_language'] = preview_lang

########NEW FILE########
__FILENAME__ = 0001_initial
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'DarkLangConfig'
        db.create_table('dark_lang_darklangconfig', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('change_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
            ('changed_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, on_delete=models.PROTECT)),
            ('enabled', self.gf('django.db.models.fields.BooleanField')(default=False)),
            ('released_languages', self.gf('django.db.models.fields.TextField')(blank=True)),
        ))
        db.send_create_signal('dark_lang', ['DarkLangConfig'])


    def backwards(self, orm):
        # Deleting model 'DarkLangConfig'
        db.delete_table('dark_lang_darklangconfig')


    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'dark_lang.darklangconfig': {
            'Meta': {'object_name': 'DarkLangConfig'},
            'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}),
            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'released_languages': ('django.db.models.fields.TextField', [], {'blank': 'True'})
        }
    }

    complete_apps = ['dark_lang']
########NEW FILE########
__FILENAME__ = 0002_enable_on_install
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        """
        Enable DarkLang by default when it is installed, to prevent accidental
        release of testing languages.
        """
        orm.DarkLangConfig(enabled=True).save()

    def backwards(self, orm):
        "Write your backwards methods here."
        raise RuntimeError("Cannot reverse this migration.")

    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'dark_lang.darklangconfig': {
            'Meta': {'object_name': 'DarkLangConfig'},
            'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}),
            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'released_languages': ('django.db.models.fields.TextField', [], {'blank': 'True'})
        }
    }

    complete_apps = ['dark_lang']
    symmetrical = True

########NEW FILE########
__FILENAME__ = models
"""
Models for the dark-launching languages
"""
from django.db import models

from config_models.models import ConfigurationModel


class DarkLangConfig(ConfigurationModel):
    """
    Configuration for the dark_lang django app
    """
    released_languages = models.TextField(
        blank=True,
        help_text="A comma-separated list of language codes to release to the public."
    )

    @property
    def released_languages_list(self):
        """
        ``released_languages`` as a list of language codes.

        Example: ['it', 'de-at', 'es', 'pt-br']
        """
        if not self.released_languages.strip():  # pylint: disable=no-member
            return []

        languages = [lang.strip() for lang in self.released_languages.split(',')]  # pylint: disable=no-member
        # Put in alphabetical order
        languages.sort()
        return languages

########NEW FILE########
__FILENAME__ = tests
"""
Tests of DarkLangMiddleware
"""
from django.contrib.auth.models import User
from django.http import HttpRequest

from django.test import TestCase
from mock import Mock

from dark_lang.middleware import DarkLangMiddleware
from dark_lang.models import DarkLangConfig


UNSET = object()


def set_if_set(dct, key, value):
    """
    Sets ``key`` in ``dct`` to ``value``
    unless ``value`` is ``UNSET``
    """
    if value is not UNSET:
        dct[key] = value


class DarkLangMiddlewareTests(TestCase):
    """
    Tests of DarkLangMiddleware
    """
    def setUp(self):
        self.user = User()
        self.user.save()
        DarkLangConfig(
            released_languages='rel',
            changed_by=self.user,
            enabled=True
        ).save()

    def process_request(self, django_language=UNSET, accept=UNSET, preview_lang=UNSET, clear_lang=UNSET):
        """
        Build a request and then process it using the ``DarkLangMiddleware``.

        Args:
            django_language (str): The language code to set in request.session['django_language']
            accept (str): The accept header to set in request.META['HTTP_ACCEPT_LANGUAGE']
            preview_lang (str): The value to set in request.GET['preview_lang']
            clear_lang (str): The value to set in request.GET['clear_lang']
        """
        session = {}
        set_if_set(session, 'django_language', django_language)

        meta = {}
        set_if_set(meta, 'HTTP_ACCEPT_LANGUAGE', accept)

        get = {}
        set_if_set(get, 'preview-lang', preview_lang)
        set_if_set(get, 'clear-lang', clear_lang)

        request = Mock(
            spec=HttpRequest,
            session=session,
            META=meta,
            GET=get
        )
        self.assertIsNone(DarkLangMiddleware().process_request(request))
        return request

    def assertAcceptEquals(self, value, request):
        """
        Assert that the HTML_ACCEPT_LANGUAGE header in request
        is equal to value
        """
        self.assertEquals(
            value,
            request.META.get('HTTP_ACCEPT_LANGUAGE', UNSET)
        )

    def test_empty_accept(self):
        self.assertAcceptEquals(UNSET, self.process_request())

b9a0e8fe37f45d402efd7db7d987a5c13deb925e
__FILENAME__ = context_processors

import ConfigParser
from django.conf import settings
import logging


log = logging.getLogger(__name__)


# Open and parse the configuration file when the module is initialized
config_file = open(settings.REPO_ROOT / "docs" / "config.ini")
config = ConfigParser.ConfigParser()
config.readfp(config_file)


def doc_url(request=None):  # pylint: disable=unused-argument
    """
    This function is added in the list of TEMPLATE_CONTEXT_PROCESSORS, which is a django setting for
    a tuple of callables that take a request object as their argument and return a dictionary of items
    to be merged into the RequestContext.

    This function returns a dict with get_online_help_info, making it directly available to all mako templates.

    Args:
        request: Currently not used, but is passed by django to context processors.
            May be used in the future for determining the language of choice.
    """

    def get_online_help_info(page_token=None):
        """
        Args:
            page_token: A string that identifies the page for which the help information is requested.
                It should correspond to an option in the docs/config.ini file.  If it doesn't, the "default"
                option is used instead.

        Returns:
            A dict mapping the following items
                * "doc_url" - a string with the url corresponding to the online help location for the given page_token.
                * "pdf_url" - a string with the url corresponding to the location of the PDF help file.
        """

        def get_config_value_with_default(section_name, option, default_option="default"):
            """
            Args:
                section_name: name of the section in the configuration from which the option should be found
                option: name of the configuration option
                default_option: name of the default configuration option whose value should be returned if the
                    requested option is not found
            """
            try:
                return config.get(section_name, option)
            except (ConfigParser.NoOptionError, AttributeError):
                log.debug("Didn't find a configuration option for '%s' section and '%s' option", section_name, option)
                return config.get(section_name, default_option)

        def get_doc_url():
            """
            Returns:
                The URL for the documentation
            """
            return "{url_base}/{language}/{version}/{page_path}".format(
                url_base=config.get("help_settings", "url_base"),
                language=get_config_value_with_default("locales", settings.LANGUAGE_CODE),
                version=config.get("help_settings", "version"),
                page_path=get_config_value_with_default("pages", page_token),
            )

        def get_pdf_url():
            """
            Returns:
                The URL for the PDF document using the pdf_settings and the help_settings (version) in the configuration
            """
            return "{pdf_base}/{version}/{pdf_file}".format(
                pdf_base=config.get("pdf_settings", "pdf_base"),
                version=config.get("help_settings", "version"),
                pdf_file=config.get("pdf_settings", "pdf_file"),
            )

        return {
            "doc_url": get_doc_url(),
            "pdf_url": get_pdf_url(),
        }

    return {'get_online_help_info': get_online_help_info}

########NEW FILE########
__FILENAME__ = course_info_model
"""
Views for viewing, adding, updating and deleting course updates.

Current db representation:
{
    "_id" : locationjson,
    "definition" : {
        "data" : "<ol>[<li><h2>date</h2>content</li>]</ol>"},
        "items" : [{"id": ID, "date": DATE, "content": CONTENT}]
        "metadata" : ignored
    }
}
"""

import re
import logging

from django.http import HttpResponseBadRequest
import django.utils
from django.utils.translation import ugettext as _
from lxml import html, etree

from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.html_module import CourseInfoModule

# # This should be in a class which inherits from XmlDescriptor
log = logging.getLogger(__name__)


def get_course_updates(location, provided_id):
    """
    Retrieve the relevant course_info updates and unpack into the model which the client expects:
    [{id : index, date : string, content : html string}]
    """
    try:
        course_updates = modulestore('direct').get_item(location)
    except ItemNotFoundError:
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)

    course_update_items = get_course_update_items(course_updates, provided_id)
    return _get_visible_update(course_update_items)


def update_course_updates(location, update, passed_id=None, user=None):
    """
    Either add or update the given course update. It will add it if the passed_id is absent or None. It will update it if
    it has an passed_id which has a valid value. Until updates have distinct values, the passed_id is the location url + an index
    into the html structure.
    """
    try:
        course_updates = modulestore('direct').get_item(location)
    except ItemNotFoundError:
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)

    course_update_items = list(reversed(get_course_update_items(course_updates)))

    if passed_id is not None:
        passed_index = _get_index(passed_id)
        # oldest update at start of list
        if 0 < passed_index <= len(course_update_items):
            course_update_dict = course_update_items[passed_index - 1]
            course_update_dict["date"] = update["date"]
            course_update_dict["content"] = update["content"]
            course_update_items[passed_index - 1] = course_update_dict
        else:
            return HttpResponseBadRequest(_("Invalid course update id."))
    else:
        course_update_dict = {
            "id": len(course_update_items) + 1,
            "date": update["date"],
            "content": update["content"],
            "status": CourseInfoModule.STATUS_VISIBLE
        }
        course_update_items.append(course_update_dict)

    # update db record
    save_course_update_items(location, course_updates, course_update_items, user)
    # remove status key
    if "status" in course_update_dict:
        del course_update_dict["status"]
    return course_update_dict


def _course_info_content(html_parsed):
    """
    Constructs the HTML for the course info update, not including the header.
    """
    if len(html_parsed) == 1:
        # could enforce that update[0].tag == 'h2'
        content = html_parsed[0].tail
    else:
        content = html_parsed[0].tail if html_parsed[0].tail is not None else ""
        content += "\n".join([html.tostring(ele) for ele in html_parsed[1:]])
    return content


def _make_update_dict(update):
    """
    Return course update item as a dictionary with required keys ('id', "date" and "content").
    """
    return {
        "id": update["id"],
        "date": update["date"],
        "content": update["content"],
    }


def _get_visible_update(course_update_items):
    """
    Filter course update items which have status "deleted".
    """
    if isinstance(course_update_items, dict):
        # single course update item
        if course_update_items.get("status") != CourseInfoModule.STATUS_DELETED:
            return _make_update_dict(course_update_items)
        else:
            # requested course update item has been deleted (soft delete)
            return {"error": _("Course update not found."), "status": 404}

    return ([_make_update_dict(update) for update in course_update_items
             if update.get("status") != CourseInfoModule.STATUS_DELETED])


# pylint: disable=unused-argument
def delete_course_update(location, update, passed_id, user):
    """
    Don't delete course update item from db.
    Delete the given course_info update by settings "status" flag to 'deleted'.
    Returns the resulting course_updates.
    """
    if not passed_id:
        return HttpResponseBadRequest()

    try:
        course_updates = modulestore('direct').get_item(location)
    except ItemNotFoundError:
        return HttpResponseBadRequest()

    course_update_items = list(reversed(get_course_update_items(course_updates)))
    passed_index = _get_index(passed_id)

    # delete update item from given index
    if 0 < passed_index <= len(course_update_items):
        course_update_item = course_update_items[passed_index - 1]
        # soft delete course update item
        course_update_item["status"] = CourseInfoModule.STATUS_DELETED
        course_update_items[passed_index - 1] = course_update_item

        # update db record
        save_course_update_items(location, course_updates, course_update_items, user)
        return _get_visible_update(course_update_items)
    else:
        return HttpResponseBadRequest(_("Invalid course update id."))


def _get_index(passed_id=None):
    """
    From the url w/ index appended, get the index.
    """
    if passed_id:
        index_matcher = re.search(r'.*?/?(\d+)$', passed_id)
        if index_matcher:
            return int(index_matcher.group(1))

    # return 0 if no index found
    return 0


def get_course_update_items(course_updates, provided_id=None):
    """
    Returns list of course_updates data dictionaries either from new format if available or
    from old. This function don't modify old data to new data (in db), instead returns data
    in common old dictionary format.
    New Format: {"items" : [{"id": computed_id, "date": date, "content": html-string}],
                 "data": "<ol>[<li><h2>date</h2>content</li>]</ol>"}
    Old Format: {"data": "<ol>[<li><h2>date</h2>content</li>]</ol>"}
    """
    if course_updates and getattr(course_updates, "items", None):
        provided_id = _get_index(provided_id)
        if provided_id and 0 < provided_id <= len(course_updates.items):
            return course_updates.items[provided_id - 1]

        # return list in reversed order (old format: [4,3,2,1]) for compatibility
        return list(reversed(course_updates.items))
    else:
        # old method to get course updates
        # purely to handle free formed updates not done via editor. Actually kills them, but at least doesn't break.
        try:
            course_html_parsed = html.fromstring(course_updates.data)
        except (etree.XMLSyntaxError, etree.ParserError):
            log.error("Cannot parse: " + course_updates.data)
            escaped = django.utils.html.escape(course_updates.data)
            course_html_parsed = html.fromstring("<ol><li>" + escaped + "</li></ol>")

        # confirm that root is <ol>, iterate over <li>, pull out <h2> subs and then rest of val
        course_update_items = []
        provided_id = _get_index(provided_id)
        if course_html_parsed.tag == 'ol':
            # 0 is the newest
            for index, update in enumerate(course_html_parsed):
                if len(update) > 0:
                    content = _course_info_content(update)
                    # make the id on the client be 1..len w/ 1 being the oldest and len being the newest
                    computed_id = len(course_html_parsed) - index
                    payload = {
                        "id": computed_id,
                        "date": update.findtext("h2"),
                        "content": content
                    }
                    if provided_id == 0:
                        course_update_items.append(payload)
                    elif provided_id == computed_id:
                        return payload

        return course_update_items


def _get_html(course_updates_items):
    """
    Method to create course_updates_html from course_updates items
    """
    list_items = []
    for update in reversed(course_updates_items):
        # filter course update items which have status "deleted".
        if update.get("status") != CourseInfoModule.STATUS_DELETED:
            list_items.append(u"<article><h2>{date}</h2>{content}</article>".format(**update))
    return u"<section>{list_items}</section>".format(list_items="".join(list_items))


def save_course_update_items(location, course_updates, course_update_items, user=None):
    """
    Save list of course_updates data dictionaries in new field ("course_updates.items")
    and html related to course update in 'data' ("course_updates.data") field.
    """
    course_updates.items = course_update_items
    course_updates.data = _get_html(course_update_items)

    # update db record
    modulestore('direct').update_item(course_updates, user)

    return course_updates

########NEW FILE########
__FILENAME__ = debug_file_uploader
from django.core.files.uploadhandler import FileUploadHandler
import time


class DebugFileUploader(FileUploadHandler):
    def __init__(self, request=None):
        super(DebugFileUploader, self).__init__(request)
        self.count = 0

    def receive_data_chunk(self, raw_data, start):
        time.sleep(1)
        self.count = self.count + len(raw_data)
        fail_at = None
        if 'fail_at' in self.request.GET:
            fail_at = int(self.request.GET.get('fail_at'))
        if fail_at and self.count > fail_at:
            raise Exception('Triggered fail')

        return raw_data

    def file_complete(self, file_size):
        return None

########NEW FILE########
__FILENAME__ = advanced_settings
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from nose.tools import assert_false, assert_equal, assert_regexp_matches  # pylint: disable=E0611
from common import type_in_codemirror, press_the_notification_button, get_codemirror_value

KEY_CSS = '.key input.policy-key'
DISPLAY_NAME_KEY = "display_name"
DISPLAY_NAME_VALUE = '"Robot Super Course"'


@step('I select the Advanced Settings$')
def i_select_advanced_settings(step):

    world.click_course_settings()

    # The click handlers are set up so that if you click <body>
    # the menu disappears.  This means that if we're even a *little*
    # bit off on the last item ('Advanced Settings'), the menu
    # will close and the test will fail.
    # For this reason, we retrieve the link and visit it directly
    # This is what the browser *should* be doing, since it's just a native
    # link with no JavaScript involved.
    link_css = 'li.nav-course-settings-advanced a'
    world.wait_for_visible(link_css)
    link = world.css_find(link_css).first['href']
    world.visit(link)


@step('I am on the Advanced Course Settings page in Studio$')
def i_am_on_advanced_course_settings(step):
    step.given('I have opened a new course in Studio')
    step.given('I select the Advanced Settings')


@step(u'I edit the value of a policy key$')
def edit_the_value_of_a_policy_key(step):
    type_in_codemirror(get_index_of(DISPLAY_NAME_KEY), 'X')


@step(u'I edit the value of a policy key and save$')
def edit_the_value_of_a_policy_key_and_save(step):
    change_display_name_value(step, '"foo"')


@step('I create a JSON object as a value for "(.*)"$')
def create_JSON_object(step, key):
    change_value(step, key, '{"key": "value", "key_2": "value_2"}')


@step('I create a non-JSON value not in quotes$')
def create_value_not_in_quotes(step):
    change_display_name_value(step, 'quote me')


@step('I see default advanced settings$')
def i_see_default_advanced_settings(step):
    # Test only a few of the existing properties (there are around 34 of them)
    assert_policy_entries(
        ["advanced_modules", DISPLAY_NAME_KEY, "show_calculator"], ["[]", DISPLAY_NAME_VALUE, "false"])


@step('the settings are alphabetized$')
def they_are_alphabetized(step):
    key_elements = world.css_find(KEY_CSS)
    all_keys = []
    for key in key_elements:
        all_keys.append(key.value)

    assert_equal(sorted(all_keys), all_keys, "policy keys were not sorted")


@step('it is displayed as formatted$')
def it_is_formatted(step):
    assert_policy_entries(['discussion_topics'], ['{\n    "key": "value",\n    "key_2": "value_2"\n}'])


@step('I get an error on save$')
def error_on_save(step):
    assert_regexp_matches(world.css_text('#notification-error-description'), 'Incorrect setting format')


@step('it is displayed as a string')
def it_is_displayed_as_string(step):
    assert_policy_entries([DISPLAY_NAME_KEY], ['"quote me"'])


@step(u'the policy key value is unchanged$')
def the_policy_key_value_is_unchanged(step):
    assert_equal(get_display_name_value(), DISPLAY_NAME_VALUE)


@step(u'the policy key value is changed$')
def the_policy_key_value_is_changed(step):
    assert_equal(get_display_name_value(), '"foo"')


def assert_policy_entries(expected_keys, expected_values):
    for key, value in zip(expected_keys, expected_values):
        index = get_index_of(key)
        assert_false(index == -1, "Could not find key: {key}".format(key=key))
        found_value = get_codemirror_value(index)
        assert_equal(
            value, found_value,
            "Expected {} to have value {} but found {}".format(key, value, found_value)
        )


def get_index_of(expected_key):
    for i, element in enumerate(world.css_find(KEY_CSS)):
        # Sometimes get stale reference if I hold on to the array of elements
        key = world.css_value(KEY_CSS, index=i)
        if key == expected_key:
            return i

    return -1


def get_display_name_value():
    index = get_index_of(DISPLAY_NAME_KEY)
    return get_codemirror_value(index)

def change_display_name_value(step, new_value):
    change_value(step, DISPLAY_NAME_KEY, new_value)

def change_value(step, key, new_value):
    index = get_index_of(key)
    type_in_codemirror(index, new_value)
    press_the_notification_button(step, "Save")
    world.wait_for_ajax_complete()

########NEW FILE########
__FILENAME__ = checklists
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from nose.tools import assert_true, assert_equal  # pylint: disable=E0611
from terrain.steps import reload_the_page
from selenium.common.exceptions import StaleElementReferenceException


############### ACTIONS ####################
@step('I select Checklists from the Tools menu$')
def i_select_checklists(step):
    world.click_tools()
    link_css = 'li.nav-course-tools-checklists a'
    world.css_click(link_css)
    world.wait_for_ajax_complete()


@step('I have opened Checklists$')
def i_have_opened_checklists(step):
    step.given('I have opened a new course in Studio')
    step.given('I select Checklists from the Tools menu')


@step('I see the four default edX checklists$')
def i_see_default_checklists(step):
    checklists = world.css_find('.checklist-title')
    assert_equal(4, len(checklists))
    assert_true(checklists[0].text.endswith('Getting Started With Studio'))
    assert_true(checklists[1].text.endswith('Draft a Rough Course Outline'))
    assert_true(checklists[2].text.endswith("Explore edX\'s Support Tools"))
    assert_true(checklists[3].text.endswith('Draft Your Course About Page'))


@step('I can check and uncheck tasks in a checklist$')
def i_can_check_and_uncheck_tasks(step):
    # Use the 2nd checklist as a reference
    verifyChecklist2Status(0, 7, 0)
    toggleTask(1, 0)
    verifyChecklist2Status(1, 7, 14)
    toggleTask(1, 3)
    verifyChecklist2Status(2, 7, 29)
    toggleTask(1, 6)
    verifyChecklist2Status(3, 7, 43)
    toggleTask(1, 3)
    verifyChecklist2Status(2, 7, 29)


@step('the tasks are correctly selected$')
def tasks_correctly_selected(step):
    verifyChecklist2Status(2, 7, 29)
    # verify that task 7 is still selected by toggling its checkbox state and making sure that it deselects
    world.browser.execute_script("window.scrollBy(0,1000)")
    toggleTask(1, 6)
    verifyChecklist2Status(1, 7, 14)


@step('I select a link to the course outline$')
def i_select_a_link_to_the_course_outline(step):
    clickActionLink(1, 0, 'Edit Course Outline')


@step('I am brought to the course outline page$')
def i_am_brought_to_course_outline(step):
    assert world.is_css_present('body.view-outline')
    assert_equal(1, len(world.browser.windows))


@step('I am brought back to the course outline in the correct state$')
def i_am_brought_back_to_course_outline(step):
    step.given('I see the four default edX checklists')
    # In a previous step, we selected (1, 0) in order to click the 'Edit Course Outline' link.
    # Make sure the task is still showing as selected (there was a caching bug with the collection).
    verifyChecklist2Status(1, 7, 14)


@step('I select a link to help page$')
def i_select_a_link_to_the_help_page(step):
    clickActionLink(2, 0, 'Visit Studio Help')


@step('I am brought to the help page in a new window$')
def i_am_brought_to_help_page_in_new_window(step):
    step.given('I see the four default edX checklists')
    windows = world.browser.windows
    assert_equal(2, len(windows))
    world.browser.switch_to_window(windows[1])
    assert_equal('http://help.edge.edx.org/', world.browser.url)


############### HELPER METHODS ####################
def verifyChecklist2Status(completed, total, percentage):
    def verify_count(driver):
        try:
            statusCount = world.css_find('#course-checklist1 .status-count').first
            return statusCount.text == str(completed)
        except StaleElementReferenceException:
            return False

    world.wait_for(verify_count)
    assert_equal(str(total), world.css_find('#course-checklist1 .status-amount').first.text)
    # Would like to check the CSS width, but not sure how to do that.
    assert_equal(str(percentage), world.css_find('#course-checklist1 .viz-checklist-status-value .int').first.text)


def toggleTask(checklist, task):
    world.css_click('#course-checklist' + str(checklist) + '-task' + str(task))
    world.wait_for_ajax_complete()


# TODO: figure out a way to do this in phantom and firefox
# For now we will mark the scenerios that use this method as skipped
def clickActionLink(checklist, task, actionText):
    # text will be empty initially, wait for it to populate
    def verify_action_link_text(driver):
        actualText = world.css_text('#course-checklist' + str(checklist) + ' a', index=task)
        if actualText == actionText:
            return True
        else:
            # toggle checklist item to make sure that the link button is showing
            toggleTask(checklist, task)
            return False

    world.wait_for(verify_action_link_text)
    world.css_click('#course-checklist' + str(checklist) + ' a', index=task)
    world.wait_for_ajax_complete()

########NEW FILE########
__FILENAME__ = common
# pylint: disable=C0111
# pylint: disable=W0621

import os
from lettuce import world, step
from nose.tools import assert_true, assert_in  # pylint: disable=no-name-in-module
from django.conf import settings

from student.roles import CourseStaffRole, CourseInstructorRole, GlobalStaff
from student.models import get_user

from selenium.webdriver.common.keys import Keys

from logging import getLogger
from student.tests.factories import AdminFactory
from student import auth
logger = getLogger(__name__)

from terrain.browser import reset_data

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT


@step('I (?:visit|access|open) the Studio homepage$')
def i_visit_the_studio_homepage(_step):
    # To make this go to port 8001, put
    # LETTUCE_SERVER_PORT = 8001
    # in your settings.py file.
    world.visit('/')
    signin_css = 'a.action-signin'
    assert world.is_css_present(signin_css)


@step('I am logged into Studio$')
def i_am_logged_into_studio(_step):
    log_into_studio()


@step('I confirm the alert$')
def i_confirm_with_ok(_step):
    world.browser.get_alert().accept()


@step(u'I press the "([^"]*)" delete icon$')
def i_press_the_category_delete_icon(_step, category):
    if category == 'section':
        css = 'a.action.delete-section-button'
    elif category == 'subsection':
        css = 'a.action.delete-subsection-button'
    else:
        assert False, 'Invalid category: %s' % category
    world.css_click(css)


@step('I have opened a new course in Studio$')
def i_have_opened_a_new_course(_step):
    open_new_course()


@step('(I select|s?he selects) the new course')
def select_new_course(_step, whom):
    course_link_css = 'a.course-link'
    world.css_click(course_link_css)


@step(u'I press the "([^"]*)" notification button$')
def press_the_notification_button(_step, name):

    # Because the notification uses a CSS transition,
    # Selenium will always report it as being visible.
    # This makes it very difficult to successfully click
    # the "Save" button at the UI level.
    # Instead, we use JavaScript to reliably click
    # the button.
    btn_css = 'div#page-notification a.action-%s' % name.lower()
    world.trigger_event(btn_css, event='focus')
    world.browser.execute_script("$('{}').click()".format(btn_css))
    world.wait_for_ajax_complete()


@step('I change the "(.*)" field to "(.*)"$')
def i_change_field_to_value(_step, field, value):
    field_css = '#%s' % '-'.join([s.lower() for s in field.split()])
    ele = world.css_find(field_css).first
    ele.fill(value)
    ele._element.send_keys(Keys.ENTER)


@step('I reset the database')
def reset_the_db(_step):
    """
    When running Lettuce tests using examples (i.e. "Confirmation is
    shown on save" in course-settings.feature), the normal hooks
    aren't called between examples. reset_data should run before each
    scenario to flush the test database. When this doesn't happen we
    get errors due to trying to insert a non-unique entry. So instead,
    we delete the database manually. This has the effect of removing
    any users and courses that have been created during the test run.
    """
    reset_data(None)


@step('I see a confirmation that my changes have been saved')
def i_see_a_confirmation(step):
    confirmation_css = '#alert-confirmation'
    assert world.is_css_present(confirmation_css)


def open_new_course():
    world.clear_courses()
    create_studio_user()
    log_into_studio()
    create_a_course()


def create_studio_user(
        uname='robot',
        email='robot+studio@edx.org',
        password='test',
        is_staff=False):
    studio_user = world.UserFactory(
        username=uname,
        email=email,
        password=password,
        is_staff=is_staff)

    registration = world.RegistrationFactory(user=studio_user)
    registration.register(studio_user)
    registration.activate()

    return studio_user


def fill_in_course_info(
        name='Robot Super Course',
        org='MITx',
        num='101',
        run='2013_Spring'):
    world.css_fill('.new-course-name', name)
    world.css_fill('.new-course-org', org)
    world.css_fill('.new-course-number', num)
    world.css_fill('.new-course-run', run)


def log_into_studio(
        uname='robot',
        email='robot+studio@edx.org',
        password='test',
        name='Robot Studio'):

    world.log_in(username=uname, password=password, email=email, name=name)
    # Navigate to the studio dashboard
    world.visit('/')
    assert_in(uname, world.css_text('h2.title', timeout=10))


def add_course_author(user, course):
    """
    Add the user to the instructor group of the course
    so they will have the permissions to see it in studio
    """
    global_admin = AdminFactory()
    for role in (CourseStaffRole, CourseInstructorRole):
        auth.add_users(global_admin, role(course.id), user)


def create_a_course():
    course = world.CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
    world.scenario_dict['COURSE'] = course

    user = world.scenario_dict.get("USER")
    if not user:
        user = get_user('robot+studio@edx.org')

    add_course_author(user, course)

    # Navigate to the studio dashboard
    world.visit('/')
    course_link_css = 'a.course-link'
    world.css_click(course_link_css)
    course_title_css = 'span.course-title'
    assert_true(world.is_css_present(course_title_css))


def add_section(name='My Section'):
    link_css = 'a.new-courseware-section-button'
    world.css_click(link_css)
    name_css = 'input.new-section-name'
    save_css = 'input.new-section-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)
    span_css = 'span.section-name-span'
    assert_true(world.is_css_present(span_css))


def add_subsection(name='Subsection One'):
    css = 'a.new-subsection-item'
    world.css_click(css)
    name_css = 'input.new-subsection-name-input'
    save_css = 'input.new-subsection-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)


def set_date_and_time(date_css, desired_date, time_css, desired_time, key=None):
    set_element_value(date_css, desired_date, key)
    world.wait_for_ajax_complete()

    set_element_value(time_css, desired_time, key)
    world.wait_for_ajax_complete()


def set_element_value(element_css, element_value, key=None):
    element = world.css_find(element_css).first
    element.fill(element_value)
    # hit TAB or provided key to trigger save content
    if key is not None:
        element._element.send_keys(getattr(Keys, key))  # pylint: disable=protected-access
    else:
        element._element.send_keys(Keys.TAB)  # pylint: disable=protected-access


@step('I have enabled the (.*) advanced module$')
def i_enabled_the_advanced_module(step, module):
    step.given('I have opened a new course section in Studio')
    world.css_click('.nav-course-settings')
    world.css_click('.nav-course-settings-advanced a')
    type_in_codemirror(0, '["%s"]' % module)
    press_the_notification_button(step, 'Save')


@world.absorb
def create_course_with_unit():
    """
    Prepare for tests by creating a course with a section, subsection, and unit.
    Performs the following:
        Clear out all courseware
        Create a course with a section, subsection, and unit
        Create a user and make that user a course author
        Log the user into studio
        Open the course from the dashboard
        Expand the section and click on the New Unit link
    The end result is the page where the user is editing the new unit
    """
    world.clear_courses()
    course = world.CourseFactory.create()
    world.scenario_dict['COURSE'] = course
    section = world.ItemFactory.create(parent_location=course.location)
    world.ItemFactory.create(
        parent_location=section.location,
        category='sequential',
        display_name='Subsection One',
    )
    user = create_studio_user(is_staff=False)
    add_course_author(user, course)

    log_into_studio()
    world.css_click('a.course-link')

    world.wait_for_js_to_load()
    css_selectors = [
        'div.section-item a.expand-collapse', 'a.new-unit-item'
    ]
    for selector in css_selectors:
        world.css_click(selector)

    world.wait_for_mathjax()
    world.wait_for_xmodule()

    assert world.is_css_present('ul.new-component-type')


@step('I have clicked the new unit button$')
@step(u'I am in Studio editing a new unit$')
def edit_new_unit(step):
    create_course_with_unit()


@step('the save notification button is disabled')
def save_button_disabled(step):
    button_css = '.action-save'
    disabled = 'is-disabled'
    assert world.css_has_class(button_css, disabled)


@step('the "([^"]*)" button is disabled')
def button_disabled(step, value):
    button_css = 'input[value="%s"]' % value
    assert world.css_has_class(button_css, 'is-disabled')


def _do_studio_prompt_action(intent, action):
    """
    Wait for a studio prompt to appear and press the specified action button
    See cms/static/js/views/feedback_prompt.js for implementation
    """
    assert intent in ['warning', 'error', 'confirmation', 'announcement',
        'step-required', 'help', 'mini']
    assert action in ['primary', 'secondary']

    world.wait_for_present('div.wrapper-prompt.is-shown#prompt-{}'.format(intent))

    action_css = 'li.nav-item > a.action-{}'.format(action)
    world.trigger_event(action_css, event='focus')
    world.browser.execute_script("$('{}').click()".format(action_css))

    world.wait_for_ajax_complete()
    world.wait_for_present('div.wrapper-prompt.is-hiding#prompt-{}'.format(intent))


@world.absorb
def confirm_studio_prompt():
    _do_studio_prompt_action('warning', 'primary')


@step('I confirm the prompt')
def confirm_the_prompt(step):
    confirm_studio_prompt()


@step(u'I am shown a prompt$')
def i_am_shown_a_notification(step):
    assert world.is_css_present('.wrapper-prompt')


def type_in_codemirror(index, text, find_prefix="$"):
    script = """
    var cm = {find_prefix}('div.CodeMirror:eq({index})').get(0).CodeMirror;
    cm.getInputField().focus();
    cm.setValue(arguments[0]);
    cm.getInputField().blur();""".format(index=index, find_prefix=find_prefix)
    world.browser.driver.execute_script(script, str(text))
    world.wait_for_ajax_complete()


def get_codemirror_value(index=0, find_prefix="$"):
    return world.browser.driver.execute_script(
        """
        return {find_prefix}('div.CodeMirror:eq({index})').get(0).CodeMirror.getValue();
        """.format(index=index, find_prefix=find_prefix)
    )



def attach_file(filename, sub_path):
    path = os.path.join(TEST_ROOT, sub_path, filename)
    world.browser.execute_script("$('input.file-input').css('display', 'block')")
    world.browser.attach_file('file', os.path.abspath(path))


def upload_file(filename, sub_path=''):
    attach_file(filename, sub_path)
    button_css = '.wrapper-modal-window-assetupload .action-upload'
    world.css_click(button_css)


@step(u'"([^"]*)" logs in$')
def other_user_login(step, name):
    step.given('I log out')
    world.visit('/')

    signin_css = 'a.action-signin'
    world.is_css_present(signin_css)
    world.css_click(signin_css)

    def fill_login_form():
        login_form = world.browser.find_by_css('form#login_form')
        login_form.find_by_name('email').fill(name + '@edx.org')
        login_form.find_by_name('password').fill("test")
        login_form.find_by_name('submit').click()
    world.retry_on_exception(fill_login_form)
    assert_true(world.is_css_present('.new-course-button'))
    world.scenario_dict['USER'] = get_user(name + '@edx.org')


@step(u'the user "([^"]*)" exists( as a course (admin|staff member|is_staff))?$')
def create_other_user(_step, name, has_extra_perms, role_name):
    email = name + '@edx.org'
    user = create_studio_user(uname=name, password="test", email=email)
    if has_extra_perms:
        if role_name == "is_staff":
            GlobalStaff().add_users(user)
        else:
            if role_name == "admin":
                # admins get staff privileges, as well
                roles = (CourseStaffRole, CourseInstructorRole)
            else:
                roles = (CourseStaffRole,)
            course_key = world.scenario_dict["COURSE"].id
            global_admin = AdminFactory()
            for role in roles:
                auth.add_users(global_admin, role(course_key), user)


@step('I log out')
def log_out(_step):
    world.visit('logout')


@step(u'I click on "edit a draft"$')
def i_edit_a_draft(_step):
    world.css_click("a.create-draft")


@step(u'I click on "replace with draft"$')
def i_replace_w_draft(_step):
    world.css_click("a.publish-draft")


@step(u'I click on "delete draft"$')
def i_delete_draft(_step):
    world.css_click("a.delete-draft")


@step(u'I publish the unit$')
def publish_unit(_step):
    world.select_option('visibility-select', 'public')


@step(u'I unpublish the unit$')
def unpublish_unit(_step):
    world.select_option('visibility-select', 'private')

########NEW FILE########
__FILENAME__ = component
# pylint: disable=C0111
# pylint: disable=W0621

# Lettuce formats proposed definitions for unimplemented steps with the
# argument name "step" instead of "_step" and pylint does not like that.
# pylint: disable=W0613

from lettuce import world, step
from nose.tools import assert_true, assert_in  # pylint: disable=E0611

DISPLAY_NAME = "Display Name"


@step(u'I add this type of single step component:$')
def add_a_single_step_component(step):
    for step_hash in step.hashes:
        component = step_hash['Component']
        assert_in(component, ['Discussion', 'Video'])

        world.create_component_instance(
            step=step,
            category='{}'.format(component.lower()),
        )


@step(u'I see this type of single step component:$')
def see_a_single_step_component(step):
    for step_hash in step.hashes:
        component = step_hash['Component']
        assert_in(component, ['Discussion', 'Video'])
        component_css = 'div.xmodule_{}Module'.format(component)
        assert_true(world.is_css_present(component_css),
                    "{} couldn't be found".format(component))


@step(u'I add this type of( Advanced)? (HTML|Problem) component:$')
def add_a_multi_step_component(step, is_advanced, category):
    for step_hash in step.hashes:
        world.create_component_instance(
            step=step,
            category='{}'.format(category.lower()),
            component_type=step_hash['Component'],
            is_advanced=bool(is_advanced),
        )


@step(u'I see (HTML|Problem) components in this order:')
def see_a_multi_step_component(step, category):

    # Wait for all components to finish rendering
    selector = 'li.component div.xblock-student_view'
    world.wait_for(lambda _: len(world.css_find(selector)) == len(step.hashes))

    for idx, step_hash in enumerate(step.hashes):

        if category == 'HTML':
            html_matcher = {
                'Text':
                    '\n    \n',
                'Announcement':
                    '<p> Words of encouragement! This is a short note that most students will read. </p>',
                'Zooming Image':
                    '<h2>ZOOMING DIAGRAMS</h2>',
                'E-text Written in LaTeX':
                    '<h2>Example: E-text page</h2>',
                'Raw HTML':
                    '<p>This template is similar to the Text template. The only difference is',
            }
            actual_html = world.css_html(selector, index=idx)
            assert_in(html_matcher[step_hash['Component']], actual_html)
        else:
            actual_text = world.css_text(selector, index=idx)
            assert_in(step_hash['Component'].upper(), actual_text)


@step(u'I see a "([^"]*)" Problem component$')
def see_a_problem_component(step, category):
    component_css = 'div.xmodule_CapaModule'
    assert_true(world.is_css_present(component_css),
                'No problem was added to the unit.')

    problem_css = 'li.component div.xblock-student_view'
    actual_text = world.css_text(problem_css)
    assert_in(category.upper(), actual_text)


@step(u'I add a "([^"]*)" "([^"]*)" component$')
def add_component_category(step, component, category):
    assert category in ('single step', 'HTML', 'Problem', 'Advanced Problem')
    given_string = 'I add this type of {} component:'.format(category)
    step.given('{}\n{}\n{}'.format(given_string, '|Component|', '|{}|'.format(component)))


@step(u'I delete all components$')
def delete_all_components(step):
    count = len(world.css_find('ol.components li.component'))
    step.given('I delete "' + str(count) + '" component')


@step(u'I delete "([^"]*)" component$')
def delete_components(step, number):
    world.wait_for_xmodule()
    delete_btn_css = 'a.delete-button'
    prompt_css = 'div#prompt-warning'
    btn_css = '{} a.button.action-primary'.format(prompt_css)
    saving_mini_css = 'div#page-notification .wrapper-notification-mini'
    for _ in range(int(number)):
        world.css_click(delete_btn_css)
        assert_true(
            world.is_css_present('{}.is-shown'.format(prompt_css)),
            msg='Waiting for the confirmation prompt to be shown')

        # Pressing the button via css was not working reliably for the last component
        # when run in Chrome.
        if world.browser.driver_name is 'Chrome':
            world.browser.execute_script("$('{}').click()".format(btn_css))
        else:
            world.css_click(btn_css)

        # Wait for the saving notification to pop up then disappear
        if world.is_css_present('{}.is-shown'.format(saving_mini_css)):
            world.css_find('{}.is-hiding'.format(saving_mini_css))


@step(u'I see no components')
def see_no_components(steps):
    assert world.is_css_not_present('li.component')


@step(u'I delete a component')
def delete_one_component(step):
    world.css_click('a.delete-button')


@step(u'I edit and save a component')
def edit_and_save_component(step):
    world.css_click('.edit-button')
    world.css_click('.save-button')


@step(u'I duplicate the (first|second|third) component$')
def duplicated_component(step, ordinal):
    ord_map = {
        "first": 0,
        "second": 1,
        "third": 2,
    }
    index = ord_map[ordinal]
    duplicate_btn_css = 'a.duplicate-button'
    world.css_click(duplicate_btn_css, int(index))


@step(u'I see a Problem component with display name "([^"]*)" in position "([^"]*)"$')
def see_component_in_position(step, display_name, index):
    component_css = 'div.xmodule_CapaModule'

    def find_problem(_driver):
        return world.css_text(component_css, int(index)).startswith(display_name.upper())

    world.wait_for(find_problem, timeout_msg='Did not find the duplicated problem')


@step(u'I see the display name is "([^"]*)"')
def check_component_display_name(step, display_name):
    label = world.css_text(".component-header")
    assert display_name == label


@step(u'I change the display name to "([^"]*)"')
def change_display_name(step, display_name):
    world.edit_component_and_select_settings()
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, display_name)
    world.save_component()


@step(u'I unset the display name')
def unset_display_name(step):
    world.edit_component_and_select_settings()
    world.revert_setting_entry(DISPLAY_NAME)
    world.save_component()

########NEW FILE########
__FILENAME__ = component_settings_editor_helpers
# disable missing docstring
# pylint: disable=C0111

from lettuce import world
from nose.tools import assert_equal, assert_in  # pylint: disable=E0611
from terrain.steps import reload_the_page
from common import type_in_codemirror
from selenium.webdriver.common.keys import Keys
from cms.envs.common import FEATURES


@world.absorb
def create_component_instance(step, category, component_type=None, is_advanced=False):
    """
    Create a new component in a Unit.

    Parameters
    ----------
    category: component type (discussion, html, problem, video)
    component_type: for components with multiple templates, the link text in the menu
    is_advanced: for problems, is the desired component under the advanced menu?
    """
    assert_in(category, ['problem', 'html', 'video', 'discussion'])

    component_button_css = 'span.large-{}-icon'.format(category.lower())
    if category == 'problem':
        module_css = 'div.xmodule_CapaModule'
    else:
        module_css = 'div.xmodule_{}Module'.format(category.title())

    # Count how many of that module is on the page. Later we will
    # assert that one more was added.
    # We need to use world.browser.find_by_css instead of world.css_find
    # because it's ok if there are currently zero of them.
    module_count_before =  len(world.browser.find_by_css(module_css))

    # Disable the jquery animation for the transition to the menus.
    world.disable_jquery_animations()
    world.css_click(component_button_css)

    if category in ('problem', 'html'):
        world.wait_for_invisible(component_button_css)
        click_component_from_menu(category, component_type, is_advanced)

    expected_count = module_count_before + 1
    world.wait_for(
        lambda _: len(world.css_find(module_css)) == expected_count,
        timeout=20
    )


@world.absorb
def click_new_component_button(step, component_button_css):
    step.given('I have clicked the new unit button')

    world.css_click(component_button_css)


def _click_advanced():
    css = 'ul.problem-type-tabs a[href="#tab2"]'
    world.css_click(css)

    # Wait for the advanced tab items to be displayed
    tab2_css = 'div.ui-tabs-panel#tab2'
    world.wait_for_visible(tab2_css)


def _find_matching_link(category, component_type):
    """
    Find the link with the specified text. There should be one and only one.
    """

    # The tab shows links for the given category
    links = world.css_find('div.new-component-{} a'.format(category))

    # Find the link whose text matches what you're looking for
    matched_links = [link for link in links if link.text == component_type]

    # There should be one and only one
    assert_equal(len(matched_links), 1)
    return matched_links[0]


def click_component_from_menu(category, component_type, is_advanced):
    """
    Creates a component for a category with more
    than one template, i.e. HTML and Problem.
    For some problem types, it is necessary to click to
    the Advanced tab.
    The component_type is the link text, e.g. "Blank Common Problem"
    """
    if is_advanced:
        # Sometimes this click does not work if you go too fast.
        world.retry_on_exception(_click_advanced,
            ignored_exceptions=AssertionError)

    # Retry this in case the list is empty because you tried too fast.
    link = world.retry_on_exception(
        lambda: _find_matching_link(category, component_type),
        ignored_exceptions=AssertionError
    )

    # Wait for the link to be clickable. If you go too fast it is not.
    world.retry_on_exception(lambda: link.click())


@world.absorb
def edit_component_and_select_settings():
    world.edit_component()
    world.ensure_settings_visible()


@world.absorb
def ensure_settings_visible():
    # Select the 'settings' tab if there is one (it isn't displayed if it is the only option)
    settings_button = world.browser.find_by_css('.settings-button')
    if len(settings_button) > 0:
        world.css_click('.settings-button')


@world.absorb
def edit_component():
    world.wait_for(lambda _driver: world.css_visible('a.edit-button'))
    world.css_click('a.edit-button')
    world.wait_for_ajax_complete()


@world.absorb
def select_editor_tab(tab_name):
    editor_tabs = world.browser.find_by_css('.editor-tabs a')
    expected_tab_text = tab_name.strip().upper()
    matching_tabs = [tab for tab in editor_tabs if tab.text.upper() == expected_tab_text]
    assert len(matching_tabs) == 1
    tab = matching_tabs[0]
    tab.click()
    world.wait_for_ajax_complete()


def enter_xml_in_advanced_problem(step, text):
    """
    Edits an advanced problem (assumes only on page),
    types the provided XML, and saves the component.
    """
    world.edit_component()
    type_in_codemirror(0, text)
    world.save_component()


@world.absorb
def verify_setting_entry(setting, display_name, value, explicitly_set):
    """
    Verify the capa module fields are set as expected in the
    Advanced Settings editor.

    Parameters
    ----------
    setting: the WebDriverElement object found in the browser
    display_name: the string expected as the label
    html: the expected field value
    explicitly_set: True if the value is expected to have been explicitly set
        for the problem, rather than derived from the defaults. This is verified
        by the existence of a "Clear" button next to the field value.
    """
    assert_equal(display_name, setting.find_by_css('.setting-label')[0].html.strip())

    # Check if the web object is a list type
    # If so, we use a slightly different mechanism for determining its value
    if setting.has_class('metadata-list-enum') or setting.has_class('metadata-dict') or setting.has_class('metadata-video-translations'):
        list_value = ', '.join(ele.value for ele in setting.find_by_css('.list-settings-item'))
        assert_equal(value, list_value)
    elif setting.has_class('metadata-videolist-enum'):
        list_value = ', '.join(ele.find_by_css('input')[0].value for ele in setting.find_by_css('.videolist-settings-item'))
        assert_equal(value, list_value)
    else:
        assert_equal(value, setting.find_by_css('.setting-input')[0].value)

    # VideoList doesn't have clear button
    if not setting.has_class('metadata-videolist-enum'):
        settingClearButton = setting.find_by_css('.setting-clear')[0]
        assert_equal(explicitly_set, settingClearButton.has_class('active'))
        assert_equal(not explicitly_set, settingClearButton.has_class('inactive'))


@world.absorb
def verify_all_setting_entries(expected_entries):
    settings = world.browser.find_by_css('.wrapper-comp-setting')
    assert_equal(len(expected_entries), len(settings))
    for (counter, setting) in enumerate(settings):
        world.verify_setting_entry(
            setting, expected_entries[counter][0],
            expected_entries[counter][1], expected_entries[counter][2]
        )


@world.absorb
def save_component():
    world.css_click("a.action-save")
    world.wait_for_ajax_complete()


@world.absorb
def save_component_and_reopen(step):
    save_component()
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes WERE persisted after Save.
    reload_the_page(step)
    edit_component_and_select_settings()


@world.absorb
def cancel_component(step):
    world.css_click("a.action-cancel")
    # We have a known issue that modifications are still shown within the edit window after cancel (though)
    # they are not persisted. Refresh the browser to make sure the changes were not persisted.
    reload_the_page(step)


@world.absorb
def revert_setting_entry(label):
    get_setting_entry(label).find_by_css('.setting-clear')[0].click()


@world.absorb
def get_setting_entry(label):
    def get_setting():
        settings = world.css_find('.wrapper-comp-setting')
        for setting in settings:
            if setting.find_by_css('.setting-label')[0].value == label:
                return setting
        return None
    return world.retry_on_exception(get_setting)


@world.absorb
def get_setting_entry_index(label):
    def get_index():
        settings = world.css_find('.metadata_edit .wrapper-comp-setting')
        for index, setting in enumerate(settings):
            if setting.find_by_css('.setting-label')[0].value == label:
                return index
        return None
    return world.retry_on_exception(get_index)


@world.absorb
def set_field_value(index, value):
    """
    Set the field to the specified value.

    Note: we cannot use css_fill here because the value is not set
    until after you move away from that field.
    Instead we will find the element, set its value, then hit the Tab key
    to get to the next field.
    """
    elem = world.css_find('.metadata_edit div.wrapper-comp-setting input.setting-input')[index]
    elem.value = value
    elem.type(Keys.TAB)

########NEW FILE########
__FILENAME__ = course-export
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

from lettuce import world, step
from component_settings_editor_helpers import enter_xml_in_advanced_problem
from nose.tools import assert_true, assert_equal
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_usage_url


@step('I go to the export page$')
def i_go_to_the_export_page(step):
    world.click_tools()
    link_css = 'li.nav-course-tools-export a'
    world.css_click(link_css)


@step('I export the course$')
def i_export_the_course(step):
    step.given('I go to the export page')
    world.css_click('a.action-export')


@step('I edit and enter bad XML$')
def i_enter_bad_xml(step):
    enter_xml_in_advanced_problem(step,
        """<problem><h1>Smallest Canvas</h1>
            <p>You want to make the smallest canvas you can.</p>
            <multiplechoiceresponse>
            <choicegroup type="MultipleChoice">
              <choice correct="false"><verbatim><canvas id="myCanvas" width = 10 height = 100> </canvas></verbatim></choice>
              <choice correct="true"><code><canvas id="myCanvas" width = 10 height = 10> </canvas></code></choice>
            </choicegroup>
            </multiplechoiceresponse>
            </problem>"""
    )


@step('I edit and enter an ampersand$')
def i_enter_an_ampersand(step):
    enter_xml_in_advanced_problem(step, "<problem>&</problem>")


@step('I get an error dialog$')
def get_an_error_dialog(step):
    assert_true(world.is_css_present("div.prompt.error"))


@step('I can click to go to the unit with the error$')
def i_click_on_error_dialog(step):
    world.click_link_by_text('Correct failed component')
    assert_true(world.css_html("span.inline-error").startswith("Problem i4x://MITx/999/problem"))
    course_key = SlashSeparatedCourseKey("MITx", "999", "Robot_Super_Course")
    # we don't know the actual ID of the vertical. So just check that we did go to a
    # vertical page in the course (there should only be one).
    vertical_usage_key = course_key.make_usage_key("vertical", "")
    vertical_url = reverse_usage_url('unit_handler', vertical_usage_key)
    assert_equal(1, world.browser.url.count(vertical_url))

########NEW FILE########
__FILENAME__ = course-overview
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from nose.tools import assert_true, assert_false, assert_equal  # pylint: disable=E0611

from logging import getLogger
logger = getLogger(__name__)


@step(u'I have a course with no sections$')
def have_a_course(step):
    world.clear_courses()
    course = world.CourseFactory.create()


@step(u'I have a course with 1 section$')
def have_a_course_with_1_section(step):
    world.clear_courses()
    course = world.CourseFactory.create()
    section = world.ItemFactory.create(parent_location=course.location)
    subsection1 = world.ItemFactory.create(
        parent_location=section.location,
        category='sequential',
        display_name='Subsection One',)


@step(u'I have a course with multiple sections$')
def have_a_course_with_two_sections(step):
    world.clear_courses()
    course = world.CourseFactory.create()
    section = world.ItemFactory.create(parent_location=course.location)
    subsection1 = world.ItemFactory.create(
        parent_location=section.location,
        category='sequential',
        display_name='Subsection One',)
    section2 = world.ItemFactory.create(
        parent_location=course.location,
        display_name='Section Two',)
    subsection2 = world.ItemFactory.create(
        parent_location=section2.location,
        category='sequential',
        display_name='Subsection Alpha',)
    subsection3 = world.ItemFactory.create(
        parent_location=section2.location,
        category='sequential',
        display_name='Subsection Beta',)


@step(u'I navigate to the course overview page$')
def navigate_to_the_course_overview_page(step):
    create_studio_user(is_staff=True)
    log_into_studio()
    course_locator = 'a.course-link'
    world.css_click(course_locator)


@step(u'I navigate to the courseware page of a course with multiple sections')
def nav_to_the_courseware_page_of_a_course_with_multiple_sections(step):
    step.given('I have a course with multiple sections')
    step.given('I navigate to the course overview page')


@step(u'I add a section')
def i_add_a_section(step):
    add_section(name='My New Section That I Just Added')


@step(u'I click the "([^"]*)" link$')
def i_click_the_text_span(step, text):
    span_locator = '.toggle-button-sections span'
    assert_true(world.browser.is_element_present_by_css(span_locator))
    # first make sure that the expand/collapse text is the one you expected
    assert_true(world.css_has_value(span_locator, text))
    world.css_click(span_locator)


@step(u'I collapse the first section$')
def i_collapse_a_section(step):
    collapse_locator = 'section.courseware-section a.collapse'
    world.css_click(collapse_locator)


@step(u'I expand the first section$')
def i_expand_a_section(step):
    expand_locator = 'section.courseware-section a.expand'
    world.css_click(expand_locator)


@step(u'I see the "([^"]*)" link$')
def i_see_the_span_with_text(step, text):
    span_locator = '.toggle-button-sections span'
    assert_true(world.css_has_value(span_locator, text))
    assert_true(world.css_visible(span_locator))


@step(u'I do not see the "([^"]*)" link$')
def i_do_not_see_the_span_with_text(step, text):
    # Note that the span will exist on the page but not be visible
    span_locator = '.toggle-button-sections span'
    assert_true(world.is_css_present(span_locator))
    assert_false(world.css_visible(span_locator))


@step(u'all sections are expanded$')
def all_sections_are_expanded(step):
    subsection_locator = 'div.subsection-list'
    subsections = world.css_find(subsection_locator)
    for index in range(len(subsections)):
        assert_true(world.css_visible(subsection_locator, index=index))


@step(u'all sections are collapsed$')
def all_sections_are_collapsed(step):
    subsection_locator = 'div.subsection-list'
    subsections = world.css_find(subsection_locator)
    for index in range(len(subsections)):
        assert_false(world.css_visible(subsection_locator, index=index))


@step(u"I change an assignment's grading status")
def change_grading_status(step):
    world.css_find('a.menu-toggle').click()
    world.css_find('.menu li').first.click()


@step(u'I reorder subsections')
def reorder_subsections(_step):
    draggable_css = '.subsection-drag-handle'
    ele = world.css_find(draggable_css).first
    ele.action_chains.drag_and_drop_by_offset(
        ele._element,
        0,
        25
    ).perform()

########NEW FILE########
__FILENAME__ = course-settings
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from terrain.steps import reload_the_page
from selenium.webdriver.common.keys import Keys
from common import type_in_codemirror, upload_file
from django.conf import settings

from nose.tools import assert_true, assert_false, assert_equal  # pylint: disable=E0611

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT

COURSE_START_DATE_CSS = "#course-start-date"
COURSE_END_DATE_CSS = "#course-end-date"
ENROLLMENT_START_DATE_CSS = "#course-enrollment-start-date"
ENROLLMENT_END_DATE_CSS = "#course-enrollment-end-date"

COURSE_START_TIME_CSS = "#course-start-time"
COURSE_END_TIME_CSS = "#course-end-time"
ENROLLMENT_START_TIME_CSS = "#course-enrollment-start-time"
ENROLLMENT_END_TIME_CSS = "#course-enrollment-end-time"

DUMMY_TIME = "15:30"
DEFAULT_TIME = "00:00"


############### ACTIONS ####################
@step('I select Schedule and Details$')
def test_i_select_schedule_and_details(step):
    world.click_course_settings()
    link_css = 'li.nav-course-settings-schedule a'
    world.css_click(link_css)
    world.wait_for_requirejs(
        ["jquery", "js/models/course",
         "js/models/settings/course_details", "js/views/settings/main"])


@step('I have set course dates$')
def test_i_have_set_course_dates(step):
    step.given('I have opened a new course in Studio')
    step.given('I select Schedule and Details')
    step.given('And I set course dates')


@step('And I set course dates$')
def test_and_i_set_course_dates(step):
    set_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    set_date_or_time(COURSE_END_DATE_CSS, '12/26/2013')
    set_date_or_time(ENROLLMENT_START_DATE_CSS, '12/1/2013')
    set_date_or_time(ENROLLMENT_END_DATE_CSS, '12/10/2013')

    set_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)
    set_date_or_time(ENROLLMENT_END_TIME_CSS, DUMMY_TIME)


@step('And I clear all the dates except start$')
def test_and_i_clear_all_the_dates_except_start(step):
    set_date_or_time(COURSE_END_DATE_CSS, '')
    set_date_or_time(ENROLLMENT_START_DATE_CSS, '')
    set_date_or_time(ENROLLMENT_END_DATE_CSS, '')


@step('Then I see cleared dates$')
def test_then_i_see_cleared_dates(step):
    verify_date_or_time(COURSE_END_DATE_CSS, '')
    verify_date_or_time(ENROLLMENT_START_DATE_CSS, '')
    verify_date_or_time(ENROLLMENT_END_DATE_CSS, '')

    verify_date_or_time(COURSE_END_TIME_CSS, '')
    verify_date_or_time(ENROLLMENT_START_TIME_CSS, '')
    verify_date_or_time(ENROLLMENT_END_TIME_CSS, '')

    # Verify course start date (required) and time still there
    verify_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)


@step('I clear the course start date$')
def test_i_clear_the_course_start_date(step):
    set_date_or_time(COURSE_START_DATE_CSS, '')


@step('I receive a warning about course start date$')
def test_i_receive_a_warning_about_course_start_date(step):
    assert_true(world.css_has_text('.message-error', 'The course must have an assigned start date.'))
    assert_true('error' in world.css_find(COURSE_START_DATE_CSS).first._element.get_attribute('class'))
    assert_true('error' in world.css_find(COURSE_START_TIME_CSS).first._element.get_attribute('class'))


@step('the previously set start date is shown$')
def test_the_previously_set_start_date_is_shown(step):
    verify_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)


@step('Given I have tried to clear the course start$')
def test_i_have_tried_to_clear_the_course_start(step):
    step.given("I have set course dates")
    step.given("I clear the course start date")
    step.given("I receive a warning about course start date")


@step('I have entered a new course start date$')
def test_i_have_entered_a_new_course_start_date(step):
    set_date_or_time(COURSE_START_DATE_CSS, '12/22/2013')


@step('The warning about course start date goes away$')
def test_the_warning_about_course_start_date_goes_away(step):
    assert world.is_css_not_present('.message-error')
    assert_false('error' in world.css_find(COURSE_START_DATE_CSS).first._element.get_attribute('class'))
    assert_false('error' in world.css_find(COURSE_START_TIME_CSS).first._element.get_attribute('class'))


@step('my new course start date is shown$')
def new_course_start_date_is_shown(step):
    verify_date_or_time(COURSE_START_DATE_CSS, '12/22/2013')
    # Time should have stayed from before attempt to clear date.
    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)


@step('I change fields$')
def test_i_change_fields(step):
    set_date_or_time(COURSE_START_DATE_CSS, '7/7/7777')
    set_date_or_time(COURSE_END_DATE_CSS, '7/7/7777')
    set_date_or_time(ENROLLMENT_START_DATE_CSS, '7/7/7777')
    set_date_or_time(ENROLLMENT_END_DATE_CSS, '7/7/7777')


@step('I change the course overview')
def test_change_course_overview(_step):
    type_in_codemirror(0, "<h1>Overview</h1>")


@step('I click the "Upload Course Image" button')
def click_upload_button(_step):
    button_css = '.action-upload-image'
    world.css_click(button_css)


@step('I upload a new course image$')
def upload_new_course_image(_step):
    upload_file('image.jpg')


@step('I should see the new course image$')
def i_see_new_course_image(_step):
    img_css = '#course-image'
    images = world.css_find(img_css)
    assert len(images) == 1
    img = images[0]
    expected_src = '/c4x/MITx/999/asset/image.jpg'

    # Don't worry about the domain in the URL
    success_func = lambda _: img['src'].endswith(expected_src)
    world.wait_for(success_func)


@step('the image URL should be present in the field')
def image_url_present(_step):
    field_css = '#course-image-url'
    expected_value = '/c4x/MITx/999/asset/image.jpg'
    assert world.css_value(field_css) == expected_value


############### HELPER METHODS ####################
def set_date_or_time(css, date_or_time):
    """
    Sets date or time field.
    """
    world.css_fill(css, date_or_time)
    e = world.css_find(css).first
    # hit Enter to apply the changes
    e._element.send_keys(Keys.ENTER)


def verify_date_or_time(css, date_or_time):
    """
    Verifies date or time field.
    """
    # We need to wait for JavaScript to fill in the field, so we use
    # css_has_value(), which first checks that the field is not blank
    assert_true(world.css_has_value(css, date_or_time))


@step('I do not see the changes')
@step('I see the set dates')
def i_see_the_set_dates(_step):
    """
    Ensure that each field has the value set in `test_and_i_set_course_dates`.
    """
    verify_date_or_time(COURSE_START_DATE_CSS, '12/20/2013')
    verify_date_or_time(COURSE_END_DATE_CSS, '12/26/2013')
    verify_date_or_time(ENROLLMENT_START_DATE_CSS, '12/01/2013')
    verify_date_or_time(ENROLLMENT_END_DATE_CSS, '12/10/2013')

    verify_date_or_time(COURSE_START_TIME_CSS, DUMMY_TIME)
    # Unset times get set to 12 AM once the corresponding date has been set.
    verify_date_or_time(COURSE_END_TIME_CSS, DEFAULT_TIME)
    verify_date_or_time(ENROLLMENT_START_TIME_CSS, DEFAULT_TIME)
    verify_date_or_time(ENROLLMENT_END_TIME_CSS, DUMMY_TIME)

########NEW FILE########
__FILENAME__ = course-team
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from nose.tools import assert_in  # pylint: disable=E0611


@step(u'(I am viewing|s?he views) the course team settings$')
def view_grading_settings(_step, whom):
    world.click_course_settings()
    link_css = 'li.nav-course-settings-team a'
    world.css_click(link_css)


@step(u'I add "([^"]*)" to the course team$')
def add_other_user(_step, name):
    new_user_css = 'a.create-user-button'
    world.css_click(new_user_css)

    # Wait for the css animation to apply the is-shown class
    shown_css = 'div.wrapper-create-user.is-shown'
    world.wait_for_present(shown_css)

    email_css = 'input#user-email-input'
    world.css_fill(email_css, name + '@edx.org')
    if world.is_firefox():
        world.trigger_event(email_css)
    confirm_css = 'form.create-user button.action-primary'
    world.css_click(confirm_css)


@step(u'I delete "([^"]*)" from the course team$')
def delete_other_user(_step, name):
    to_delete_css = '.user-item .item-actions a.remove-user[data-id="{email}"]'.format(
        email="{0}{1}".format(name, '@edx.org'))
    world.css_click(to_delete_css)
    world.confirm_studio_prompt()


@step(u's?he deletes me from the course team$')
def other_delete_self(_step):
    to_delete_css = '.user-item .item-actions a.remove-user[data-id="{email}"]'.format(
        email="robot+studio@edx.org")
    world.css_click(to_delete_css)
    world.confirm_studio_prompt()


@step(u'I make "([^"]*)" a course team admin$')
def make_course_team_admin(_step, name):
    admin_btn_css = '.user-item[data-email="{name}@edx.org"] .user-actions .add-admin-role'.format(
        name=name)
    world.css_click(admin_btn_css)


@step(u'I remove admin rights from ("([^"]*)"|myself)$')
def remove_course_team_admin(_step, outer_capture, name):
    if outer_capture == "myself":
        email = world.scenario_dict["USER"].email
    else:
        email = name + '@edx.org'
    admin_btn_css = '.user-item[data-email="{email}"] .user-actions .remove-admin-role'.format(
        email=email)
    world.css_click(admin_btn_css)


@step(u'I( do not)? see the course on my page$')
@step(u's?he does( not)? see the course on (his|her) page$')
def see_course(_step, do_not_see, gender='self'):
    class_css = 'h3.course-title'
    if do_not_see:
        assert world.is_css_not_present(class_css)
    else:
        all_courses = world.css_find(class_css)
        all_names = [item.html for item in all_courses]
        assert_in(world.scenario_dict['COURSE'].display_name, all_names)


@step(u'"([^"]*)" should( not)? be marked as an admin$')
def marked_as_admin(_step, name, not_marked_admin):
    flag_css = '.user-item[data-email="{name}@edx.org"] .flag-role.flag-role-admin'.format(
        name=name)
    if not_marked_admin:
        assert world.is_css_not_present(flag_css)
    else:
        assert world.is_css_present(flag_css)


@step(u'I should( not)? be marked as an admin$')
def self_marked_as_admin(_step, not_marked_admin):
    return marked_as_admin(_step, "robot+studio", not_marked_admin)


@step(u'I can(not)? delete users$')
@step(u's?he can(not)? delete users$')
def can_delete_users(_step, can_not_delete):
    to_delete_css = 'a.remove-user'
    if can_not_delete:
        assert world.is_css_not_present(to_delete_css)
    else:
        assert world.is_css_present(to_delete_css)


@step(u'I can(not)? add users$')
@step(u's?he can(not)? add users$')
def can_add_users(_step, can_not_add):
    add_css = 'a.create-user-button'
    if can_not_add:
        assert world.is_css_not_present(add_css)
    else:
        assert world.is_css_present(add_css)


@step(u'I can(not)? make ("([^"]*)"|myself) a course team admin$')
@step(u's?he can(not)? make ("([^"]*)"|me) a course team admin$')
def can_make_course_admin(_step, can_not_make_admin, outer_capture, name):
    if outer_capture == "myself":
        email = world.scenario_dict["USER"].email
    else:
        email = name + '@edx.org'
    add_button_css = '.user-item[data-email="{email}"] .add-admin-role'.format(email=email)
    if can_not_make_admin:
        assert world.is_css_not_present(add_button_css)
    else:
        assert world.is_css_present(add_button_css)

########NEW FILE########
__FILENAME__ = course-updates
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from selenium.webdriver.common.keys import Keys
from common import type_in_codemirror, get_codemirror_value
from nose.tools import assert_in  # pylint: disable=E0611


@step(u'I go to the course updates page')
def go_to_updates(_step):
    menu_css = 'li.nav-course-courseware'
    updates_css = 'li.nav-course-courseware-updates a'
    world.css_click(menu_css)
    world.css_click(updates_css)


@step(u'I add a new update with the text "([^"]*)"$')
def add_update(_step, text):
    update_css = 'a.new-update-button'
    world.css_click(update_css)
    change_text(text)


@step(u'I should see the update "([^"]*)"$')
def check_update(_step, text):
    update_css = 'div.update-contents'
    update_html = world.css_find(update_css).html
    assert_in(text, update_html)


@step(u'I should not see the update "([^"]*)"$')
def check_no_update(_step, text):
    update_css = 'div.update-contents'
    assert world.is_css_not_present(update_css)


@step(u'I modify the text to "([^"]*)"$')
def modify_update(_step, text):
    button_css = 'div.post-preview a.edit-button'
    world.css_click(button_css)
    change_text(text)


@step(u'I change the update from "([^"]*)" to "([^"]*)"$')
def change_existing_update(_step, before, after):
    verify_text_in_editor_and_update('div.post-preview a.edit-button', before, after)


@step(u'I change the handout from "([^"]*)" to "([^"]*)"$')
def change_existing_handout(_step, before, after):
    verify_text_in_editor_and_update('div.course-handouts a.edit-button', before, after)


@step(u'I delete the update$')
def click_button(_step):
    button_css = 'div.post-preview a.delete-button'
    world.css_click(button_css)


@step(u'I edit the date to "([^"]*)"$')
def change_date(_step, new_date):
    button_css = 'div.post-preview a.edit-button'
    world.css_click(button_css)
    date_css = 'input.date'
    date = world.css_find(date_css)
    for i in range(len(date.value)):
        date._element.send_keys(Keys.END, Keys.BACK_SPACE)
    date._element.send_keys(new_date)
    save_css = 'a.save-button'
    world.css_click(save_css)


@step(u'I should see the date "([^"]*)"$')
def check_date(_step, date):
    date_css = 'span.date-display'
    assert_in(date, world.css_html(date_css))


@step(u'I modify the handout to "([^"]*)"$')
def edit_handouts(_step, text):
    edit_css = 'div.course-handouts > a.edit-button'
    world.css_click(edit_css)
    change_text(text)


@step(u'I see the handout "([^"]*)"$')
def check_handout(_step, handout):
    handout_css = 'div.handouts-content'
    assert_in(handout, world.css_html(handout_css))


@step(u'I see the handout error text')
def check_handout_error(_step):
    handout_error_css = 'div#handout_error'
    assert world.css_has_class(handout_error_css, 'is-shown')


@step(u'I see handout save button disabled')
def check_handout_error(_step):
    handout_save_button = 'form.edit-handouts-form a.save-button'
    assert world.css_has_class(handout_save_button, 'is-disabled')


@step(u'I edit the handout to "([^"]*)"$')
def edit_handouts(_step, text):
    type_in_codemirror(0, text)


@step(u'I see handout save button re-enabled')
def check_handout_error(_step):
    handout_save_button = 'form.edit-handouts-form a.save-button'
    assert not world.css_has_class(handout_save_button, 'is-disabled')


@step(u'I save handout edit')
def check_handout_error(_step):
    save_css = 'a.save-button'
    world.css_click(save_css)


def change_text(text):
    type_in_codemirror(0, text)
    save_css = 'a.save-button'
    world.css_click(save_css)


def verify_text_in_editor_and_update(button_css, before, after):
    world.css_click(button_css)
    text = get_codemirror_value()
    assert_in(before, text)
    change_text(after)

########NEW FILE########
__FILENAME__ = courses
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *

############### ACTIONS ####################


@step('There are no courses$')
def no_courses(step):
    world.clear_courses()
    create_studio_user()


@step('I click the New Course button$')
def i_click_new_course(step):
    world.css_click('.new-course-button')


@step('I fill in the new course information$')
def i_fill_in_a_new_course_information(step):
    fill_in_course_info()


@step('I create a course with "([^"]*)", "([^"]*)", "([^"]*)", and "([^"]*)"')
def i_create_course(step, name, org, number, run):
    fill_in_course_info(name=name, org=org, num=number, run=run)


@step('I create a new course$')
def i_create_a_course(step):
    create_a_course()


@step('I click the course link in My Courses$')
def i_click_the_course_link_in_my_courses(step):
    course_css = 'a.course-link'
    world.css_click(course_css)


@step('I see an error about the length of the org/course/run tuple')
def i_see_error_about_length(step):
    assert world.css_has_text('#course_creation_error', 'The combined length of the organization, course number, and course run fields cannot be more than 65 characters.')

############ ASSERTIONS ###################


@step('the Courseware page has loaded in Studio$')
def courseware_page_has_loaded_in_studio(step):
    course_title_css = 'span.course-title'
    assert world.is_css_present(course_title_css)


@step('I see the course listed in My Courses$')
def i_see_the_course_in_my_courses(step):
    course_css = 'h3.class-title'
    assert world.css_has_text(course_css, world.scenario_dict['COURSE'].display_name)


@step('I am on the "([^"]*)" tab$')
def i_am_on_tab(step, tab_name):
    header_css = 'div.inner-wrapper h1'
    assert world.css_has_text(header_css, tab_name)


@step('I see a link for adding a new section$')
def i_see_new_section_link(step):
    link_css = 'a.new-courseware-section-button'
    assert world.css_has_text(link_css, 'New Section')

########NEW FILE########
__FILENAME__ = course_import
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

import os
from lettuce import world, step
from django.conf import settings


def import_file(filename):
    world.browser.execute_script("$('input.file-input').css('display', 'block')")
    path = os.path.join(settings.COMMON_TEST_DATA_ROOT, "imports", filename)
    world.browser.attach_file('course-data', os.path.abspath(path))
    world.css_click('input.submit-button')
    # Go to course outline
    world.click_course_content()
    outline_css = 'li.nav-course-courseware-outline a'
    world.css_click(outline_css)


@step('I go to the import page$')
def go_to_import(step):
    menu_css = 'li.nav-course-tools'
    import_css = 'li.nav-course-tools-import a'
    world.css_click(menu_css)
    world.css_click(import_css)

########NEW FILE########
__FILENAME__ = discussion-editor
# disable missing docstring
# pylint: disable=C0111

from lettuce import world, step


@step('I have created a Discussion Tag$')
def i_created_discussion_tag(step):
    world.create_course_with_unit()
    world.create_component_instance(
        step=step,
        category='discussion',
    )


@step('I see three alphabetized settings and their expected values$')
def i_see_only_the_settings_and_values(step):
    world.verify_all_setting_entries(
        [
            ['Category', "Week 1", False],
            ['Display Name', "Discussion", False],
            ['Subcategory', "Topic-Level Student-Visible Label", False]
        ])


@step('I edit the component$')
def i_edit_and_select_settings(_step):
    world.edit_component()

########NEW FILE########
__FILENAME__ = grading
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from terrain.steps import reload_the_page
from selenium.common.exceptions import InvalidElementStateException
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_course_url
from nose.tools import assert_in, assert_not_in, assert_equal, assert_not_equal  # pylint: disable=E0611


@step(u'I am viewing the grading settings')
def view_grading_settings(step):
    world.click_course_settings()
    link_css = 'li.nav-course-settings-grading a'
    world.css_click(link_css)


@step(u'I add "([^"]*)" new grade')
def add_grade(step, many):
    grade_css = '.new-grade-button'
    for i in range(int(many)):
        world.css_click(grade_css)


@step(u'I delete a grade')
def delete_grade(step):
    #grade_css = 'li.grade-specific-bar > a.remove-button'
    #range_css = '.grade-specific-bar'
    #world.css_find(range_css)[1].mouseover()
    #world.css_click(grade_css)
    world.browser.execute_script('document.getElementsByClassName("remove-button")[0].click()')


@step(u'I see I now have "([^"]*)" grades$')
def view_grade_slider(step, how_many):
    grade_slider_css = '.grade-specific-bar'
    all_grades = world.css_find(grade_slider_css)
    assert_equal(len(all_grades), int(how_many))


@step(u'I move a grading section')
def move_grade_slider(step):
    moveable_css = '.ui-resizable-e'
    f = world.css_find(moveable_css).first
    f.action_chains.drag_and_drop_by_offset(f._element, 100, 0).perform()


@step(u'I see that the grade range has changed')
def confirm_change(step):
    range_css = '.range'
    all_ranges = world.css_find(range_css)
    for i in range(len(all_ranges)):
        assert_not_equal(world.css_html(range_css, index=i), '0-50')


@step(u'I change assignment type "([^"]*)" to "([^"]*)"$')
def change_assignment_name(step, old_name, new_name):
    name_id = '#course-grading-assignment-name'
    index = get_type_index(old_name)
    f = world.css_find(name_id)[index]
    assert_not_equal(index, -1)
    for count in range(len(old_name)):
        f._element.send_keys(Keys.END, Keys.BACK_SPACE)
    f._element.send_keys(new_name)


@step(u'I go back to the main course page')
def main_course_page(step):
    course_name = world.scenario_dict['COURSE'].display_name.replace(' ', '_')
    course_key = SlashSeparatedCourseKey(
        world.scenario_dict['COURSE'].org,
        world.scenario_dict['COURSE'].number,
        course_name
    )
    main_page_link = reverse_course_url('course_handler', course_key)

    world.visit(main_page_link)
    assert_in('Course Outline', world.css_text('h1.page-header'))


@step(u'I do( not)? see the assignment name "([^"]*)"$')
def see_assignment_name(step, do_not, name):
    assignment_menu_css = 'ul.menu > li > a'
    # First assert that it is there, make take a bit to redraw
    assert_true(
        world.css_find(assignment_menu_css),
        msg="Could not find assignment menu"
    )

    assignment_menu = world.css_find(assignment_menu_css)
    allnames = [item.html for item in assignment_menu]
    if do_not:
        assert_not_in(name, allnames)
    else:
        assert_in(name, allnames)


@step(u'I delete the assignment type "([^"]*)"$')
def delete_assignment_type(step, to_delete):
    delete_css = '.remove-grading-data'
    world.css_click(delete_css, index=get_type_index(to_delete))


@step(u'I add a new assignment type "([^"]*)"$')
def add_assignment_type(step, new_name):
    add_button_css = '.add-grading-data'
    world.css_click(add_button_css)
    name_id = '#course-grading-assignment-name'
    new_assignment = world.css_find(name_id)[-1]
    new_assignment._element.send_keys(new_name)


@step(u'I set the assignment weight to "([^"]*)"$')
def set_weight(step, weight):
    weight_id = '#course-grading-assignment-gradeweight'
    weight_field = world.css_find(weight_id)[-1]
    old_weight = world.css_value(weight_id, -1)
    for count in range(len(old_weight)):
        weight_field._element.send_keys(Keys.END, Keys.BACK_SPACE)
    weight_field._element.send_keys(weight)


@step(u'the assignment weight is displayed as "([^"]*)"$')
def verify_weight(step, weight):
    weight_id = '#course-grading-assignment-gradeweight'
    assert_equal(world.css_value(weight_id, -1), weight)


@step(u'I have populated the course')
def populate_course(step):
    step.given('I have added a new section')
    step.given('I have added a new subsection')


@step(u'I do not see the changes persisted on refresh$')
def changes_not_persisted(step):
    reload_the_page(step)
    name_id = '#course-grading-assignment-name'
    assert_equal(world.css_value(name_id), 'Homework')


@step(u'I see the assignment type "(.*)"$')
def i_see_the_assignment_type(_step, name):
    assignment_css = '#course-grading-assignment-name'
    assignments = world.css_find(assignment_css)
    types = [ele['value'] for ele in assignments]
    assert_in(name, types)


@step(u'I change the highest grade range to "(.*)"$')
def change_grade_range(_step, range_name):
    range_css = 'span.letter-grade'
    grade = world.css_find(range_css).first
    grade.value = range_name


@step(u'I see the highest grade range is "(.*)"$')
def i_see_highest_grade_range(_step, range_name):
    range_css = 'span.letter-grade'
    grade = world.css_find(range_css).first
    assert_equal(grade.value, range_name)


@step(u'I cannot edit the "Fail" grade range$')
def cannot_edit_fail(_step):
    range_css = 'span.letter-grade'
    ranges = world.css_find(range_css)
    assert_equal(len(ranges), 2)
    assert_not_equal(ranges.last.value, 'Failure')

    # try to change the grade range -- this should throw an exception
    try:
        ranges.last.value = 'Failure'
    except (InvalidElementStateException):
        pass  # We should get this exception on failing to edit the element

    # check to be sure that nothing has changed
    ranges = world.css_find(range_css)
    assert_equal(len(ranges), 2)
    assert_not_equal(ranges.last.value, 'Failure')


@step(u'I change the grace period to "(.*)"$')
def i_change_grace_period(_step, grace_period):
    grace_period_css = '#course-grading-graceperiod'
    ele = world.css_find(grace_period_css).first

    # Sometimes it takes a moment for the JavaScript
    # to populate the field.  If we don't wait for
    # this to happen, then we can end up with
    # an invalid value (e.g. "00:0048:00")
    # which prevents us from saving.
    assert_true(world.css_has_value(grace_period_css, "00:00"))

    # Set the new grace period
    ele.value = grace_period


@step(u'I see the grace period is "(.*)"$')
def the_grace_period_is(_step, grace_period):
    grace_period_css = '#course-grading-graceperiod'

    # The default value is 00:00
    # so we need to wait for it to change
    world.wait_for(
        lambda _: world.css_has_value(grace_period_css, grace_period)
    )


def get_type_index(name):
    name_id = '#course-grading-assignment-name'
    all_types = world.css_find(name_id)
    for index in range(len(all_types)):
        if world.css_value(name_id, index=index) == name:
            return index
    return -1

########NEW FILE########
__FILENAME__ = help
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

from nose.tools import assert_false  # pylint: disable=no-name-in-module
from lettuce import step, world


@step(u'I should see online help for "([^"]*)"$')
def see_online_help_for(step, page_name):
    # make sure the online Help link exists on this page and contains the expected page name
    elements_found = world.browser.find_by_xpath(
        '//li[contains(@class, "nav-account-help")]//a[contains(@href, "{page_name}")]'.format(
            page_name=page_name
        )
    )
    assert_false(elements_found.is_empty())

    # make sure the PDF link on the sock of this page exists
    # for now, the PDF link stays constant for all the pages so we just check for "pdf"
    elements_found = world.browser.find_by_xpath(
        '//section[contains(@class, "sock")]//li[contains(@class, "js-help-pdf")]//a[contains(@href, "pdf")]'
    )
    assert_false(elements_found.is_empty())

########NEW FILE########
__FILENAME__ = html-editor
# disable missing docstring
# pylint: disable=C0111

from lettuce import world, step
from nose.tools import assert_in, assert_false, assert_true, assert_equal  # pylint: disable=no-name-in-module
from common import type_in_codemirror, get_codemirror_value

CODEMIRROR_SELECTOR_PREFIX = "$('iframe').contents().find"


@step('I have created a Blank HTML Page$')
def i_created_blank_html_page(step):
    world.create_course_with_unit()
    world.create_component_instance(
        step=step,
        category='html',
        component_type='Text'
    )


@step('I have created a raw HTML component')
def i_created_raw_html(step):
    world.create_course_with_unit()
    world.create_component_instance(
        step=step,
        category='html',
        component_type='Raw HTML'
    )


@step('I see the HTML component settings$')
def i_see_only_the_html_display_name(step):
    world.verify_all_setting_entries(
        [
            ['Display Name', "Text", False],
            ['Editor', "Visual", False]
        ]
    )


@step('I have created an E-text Written in LaTeX$')
def i_created_etext_in_latex(step):
    world.create_course_with_unit()
    step.given('I have enabled latex compiler')
    world.create_component_instance(
        step=step,
        category='html',
        component_type='E-text Written in LaTeX'
    )


@step('I edit the page$')
def i_click_on_edit_icon(step):
    world.edit_component()


@step('I add an image with static link "(.*)" via the Image Plugin Icon$')
def i_click_on_image_plugin_icon(step, path):
    use_plugin(
        '.mce-i-image',
        lambda: world.css_fill('.mce-textbox', path, 0)
    )


@step('the link is shown as "(.*)" in the Image Plugin$')
def check_link_in_image_plugin(step, path):
    use_plugin(
        '.mce-i-image',
        lambda: assert_equal(path, world.css_find('.mce-textbox')[0].value)
    )


@step('I add a link with static link "(.*)" via the Link Plugin Icon$')
def i_click_on_link_plugin_icon(step, path):
    def fill_in_link_fields():
        world.css_fill('.mce-textbox', path, 0)
        world.css_fill('.mce-textbox', 'picture', 1)

    use_plugin('.mce-i-link', fill_in_link_fields)


@step('the link is shown as "(.*)" in the Link Plugin$')
def check_link_in_link_plugin(step, path):
    # Ensure caret position is within the link just created.
    script = """
    var editor = tinyMCE.activeEditor;
    editor.selection.select(editor.dom.select('a')[0]);"""
    world.browser.driver.execute_script(script)
    world.wait_for_ajax_complete()

    use_plugin(
        '.mce-i-link',
        lambda: assert_equal(path, world.css_find('.mce-textbox')[0].value)
    )


@step('type "(.*)" in the code editor and press OK$')
def type_in_codemirror_plugin(step, text):
    # Verify that raw code editor is not visible.
    assert_true(world.css_has_class('.CodeMirror', 'is-inactive'))
    # Verify that TinyMCE editor is present
    assert_true(world.is_css_present('.tiny-mce'))
    use_code_editor(
        lambda: type_in_codemirror(0, text, CODEMIRROR_SELECTOR_PREFIX)
    )


@step('and the code editor displays "(.*)"$')
def verify_code_editor_text(step, text):
    use_code_editor(
        lambda: assert_equal(text, get_codemirror_value(0, CODEMIRROR_SELECTOR_PREFIX))
    )


def use_plugin(button_class, action):
    # Click on plugin button
    world.css_click(button_class)
    perform_action_in_plugin(action)


def use_code_editor(action):
    # Click on plugin button
    buttons = world.css_find('div.mce-widget>button')

    code_editor = [button for button in buttons if button.text == 'HTML']
    assert_equal(1, len(code_editor))
    code_editor[0].click()

    perform_action_in_plugin(action)


def perform_action_in_plugin(action):
     # Wait for the plugin window to open.
    world.wait_for_visible('.mce-window')

    # Trigger the action
    action()

    # Click OK
    world.css_click('.mce-primary')


@step('I save the page$')
def i_click_on_save(step):
    world.save_component()


@step('the page text contains:')
def check_page_text(step):
    assert_in(step.multiline, world.css_find('.xmodule_HtmlModule').html)


@step('the Raw Editor contains exactly:')
def check_raw_editor_text(step):
    assert_equal(step.multiline, get_codemirror_value(0))


@step('the src link is rewritten to "(.*)"$')
def image_static_link_is_rewritten(step, path):
    # Find the TinyMCE iframe within the main window
    with world.browser.get_iframe('mce_0_ifr') as tinymce:
        image = tinymce.find_by_tag('img').first
        assert_in(path, image['src'])


@step('the href link is rewritten to "(.*)"$')
def link_static_link_is_rewritten(step, path):
    # Find the TinyMCE iframe within the main window
    with world.browser.get_iframe('mce_0_ifr') as tinymce:
        link = tinymce.find_by_tag('a').first
        assert_in(path, link['href'])


@step('the expected toolbar buttons are displayed$')
def check_toolbar_buttons(step):
    dropdowns = world.css_find('.mce-listbox')
    assert_equal(2, len(dropdowns))

    # Format dropdown
    assert_equal('Paragraph', dropdowns[0].text)
    # Font dropdown
    assert_equal('Font Family', dropdowns[1].text)

    buttons = world.css_find('.mce-ico')

    # Note that the code editor icon is not present because we are now showing text instead of an icon.
    # However, other test points user the code editor, so we have already verified its presence.
    expected_buttons = [
        'bold',
        'italic',
        'underline',
        'forecolor',
        # This is our custom "code style" button, which uses an image instead of a class.
        'none',
        'bullist',
        'numlist',
        'outdent',
        'indent',
        'blockquote',
        'link',
        'unlink',
        'image'
    ]

    assert_equal(len(expected_buttons), len(buttons))

    for index, button in enumerate(expected_buttons):
        class_names = buttons[index]._element.get_attribute('class')
        assert_equal("mce-ico mce-i-" + button, class_names)


@step('I set the text to "(.*)" and I select the text$')
def set_text_and_select(step, text):
    script = """
    var editor = tinyMCE.activeEditor;
    editor.setContent(arguments[0]);
    editor.selection.select(editor.dom.select('p')[0]);"""
    world.browser.driver.execute_script(script, str(text))
    world.wait_for_ajax_complete()


@step('I select the code toolbar button$')
def select_code_button(step):
    # This is our custom "code style" button. It uses an image instead of a class.
    world.css_click(".mce-i-none")


@step('type "(.*)" into the Raw Editor$')
def type_in_raw_editor(step, text):
    # Verify that CodeMirror editor is not hidden
    assert_false(world.css_has_class('.CodeMirror', 'is-inactive'))
    # Verify that TinyMCE Editor is not present
    assert_true(world.is_css_not_present('.tiny-mce'))
    type_in_codemirror(0, text)


@step('I edit the component and select the (Raw|Visual) Editor$')
def select_editor(step, editor):
    world.edit_component_and_select_settings()
    world.browser.select('Editor', editor)

########NEW FILE########
__FILENAME__ = pages
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0613

from lettuce import world, step
from nose.tools import assert_equal, assert_in  # pylint: disable=E0611


CSS_FOR_TAB_ELEMENT = "li[data-tab-id='{0}'] input.toggle-checkbox"


@step(u'I go to the pages page$')
def go_to_static(step):
    menu_css = 'li.nav-course-courseware'
    static_css = 'li.nav-course-courseware-pages a'
    world.css_click(menu_css)
    world.css_click(static_css)


@step(u'I add a new static page$')
def add_page(step):
    button_css = 'a.new-button'
    world.css_click(button_css)


@step(u'I should see a static page named "([^"]*)"$')
def see_a_static_page_named_foo(step, name):
    pages_css = 'div.xmodule_StaticTabModule'
    page_name_html = world.css_html(pages_css)
    assert_equal(page_name_html, '\n    {name}\n'.format(name=name))


@step(u'I should not see any static pages$')
def not_see_any_static_pages(step):
    pages_css = 'div.xmodule_StaticTabModule'
    assert (world.is_css_not_present(pages_css, wait_time=30))


@step(u'I "(edit|delete)" the static page$')
def click_edit_or_delete(step, edit_or_delete):
    button_css = 'ul.component-actions a.%s-button' % edit_or_delete
    world.css_click(button_css)


@step(u'I change the name to "([^"]*)"$')
def change_name(step, new_name):
    settings_css = '.settings-button'
    world.css_click(settings_css)
    input_css = 'input.setting-input'
    world.css_fill(input_css, new_name)
    if world.is_firefox():
        world.trigger_event(input_css)
    world.save_component()


@step(u'I drag the first static page to the last$')
def drag_first_static_page_to_last(step):
    drag_first_to_last_with_css('.component')


@step(u'I have created a static page$')
def create_static_page(step):
    step.given('I have opened the pages page in a new course')
    step.given('I add a new static page')


@step(u'I have opened the pages page in a new course$')
def open_pages_page_in_new_course(step):
    step.given('I have opened a new course in Studio')
    step.given('I go to the pages page')


@step(u'I have created two different static pages$')
def create_two_pages(step):
    step.given('I have created a static page')
    step.given('I "edit" the static page')
    step.given('I change the name to "First"')
    step.given('I add a new static page')
    # Verify order of pages
    _verify_page_names('First', 'Empty')


@step(u'the static pages are switched$')
def static_pages_are_switched(step):
    _verify_page_names('Empty', 'First')


def _verify_page_names(first, second):
    world.wait_for(
        func=lambda _: len(world.css_find('.xmodule_StaticTabModule')) == 2,
        timeout=200,
        timeout_msg="Timed out waiting for two pages to be present"
    )
    pages = world.css_find('.xmodule_StaticTabModule')
    assert_equal(pages[0].text, first)
    assert_equal(pages[1].text, second)


@step(u'the built-in pages are in the default order$')
def built_in_pages_in_default_order(step):
    expected_pages = ['Courseware', 'Course Info', 'Wiki', 'Progress']
    see_pages_in_expected_order(expected_pages)


@step(u'the built-in pages are switched$')
def built_in_pages_switched(step):
    expected_pages = ['Courseware', 'Course Info', 'Progress', 'Wiki']
    see_pages_in_expected_order(expected_pages)


@step(u'the pages are in the default order$')
def pages_in_default_order(step):
    expected_pages = ['Courseware', 'Course Info', 'Wiki', 'Progress', 'First', 'Empty']
    see_pages_in_expected_order(expected_pages)


@step(u'the pages are switched$$')
def pages_are_switched(step):
    expected_pages = ['Courseware', 'Course Info', 'Progress', 'First', 'Empty', 'Wiki']
    see_pages_in_expected_order(expected_pages)


@step(u'I drag the first page to the last$')
def drag_first_page_to_last(step):
    drag_first_to_last_with_css('.is-movable')


@step(u'I should see the "([^"]*)" page as "(visible|hidden)"$')
def page_is_visible_or_hidden(step, page_id, visible_or_hidden):
    hidden = visible_or_hidden == "hidden"
    assert_equal(world.css_find(CSS_FOR_TAB_ELEMENT.format(page_id)).checked, hidden)


@step(u'I toggle the visibility of the "([^"]*)" page$')
def page_toggle_visibility(step, page_id):
    world.css_find(CSS_FOR_TAB_ELEMENT.format(page_id))[0].click()


def drag_first_to_last_with_css(css_class):
    # For some reason, the drag_and_drop method did not work in this case.
    draggables = world.css_find(css_class + ' .drag-handle')
    source = draggables.first
    target = draggables.last
    source.action_chains.click_and_hold(source._element).perform()  # pylint: disable=protected-access
    source.action_chains.move_to_element_with_offset(target._element, 0, 50).perform()  # pylint: disable=protected-access
    source.action_chains.release().perform()


def see_pages_in_expected_order(page_names_in_expected_order):
    pages = world.css_find("li.course-tab")
    assert_equal(len(page_names_in_expected_order), len(pages))
    for i, page_name in enumerate(page_names_in_expected_order):
        assert_in(page_name, pages[i].text)

########NEW FILE########
__FILENAME__ = problem-editor
# disable missing docstring
# pylint: disable=C0111

import json
from lettuce import world, step
from nose.tools import assert_equal, assert_true  # pylint: disable=E0611
from common import type_in_codemirror, open_new_course
from advanced_settings import change_value
from course_import import import_file

DISPLAY_NAME = "Display Name"
MAXIMUM_ATTEMPTS = "Maximum Attempts"
PROBLEM_WEIGHT = "Problem Weight"
RANDOMIZATION = 'Randomization'
SHOW_ANSWER = "Show Answer"
TIMER_BETWEEN_ATTEMPTS = "Timer Between Attempts"

@step('I have created a Blank Common Problem$')
def i_created_blank_common_problem(step):
    world.create_course_with_unit()
    step.given("I have created another Blank Common Problem")


@step('I have created another Blank Common Problem$')
def i_create_new_common_problem(step):
    world.create_component_instance(
        step=step,
        category='problem',
        component_type='Blank Common Problem'
    )


@step('I edit and select Settings$')
def i_edit_and_select_settings(_step):
    world.edit_component_and_select_settings()


@step('I see the advanced settings and their expected values$')
def i_see_advanced_settings_with_values(step):
    world.verify_all_setting_entries(
        [
            [DISPLAY_NAME, "Blank Common Problem", True],
            [MAXIMUM_ATTEMPTS, "", False],
            [PROBLEM_WEIGHT, "", False],
            [RANDOMIZATION, "Never", False],
            [SHOW_ANSWER, "Finished", False],
            [TIMER_BETWEEN_ATTEMPTS, "0", False]
        ])


@step('I can modify the display name')
def i_can_modify_the_display_name(_step):
    # Verifying that the display name can be a string containing a floating point value
    # (to confirm that we don't throw an error because it is of the wrong type).
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, '3.4')
    verify_modified_display_name()


@step('my display name change is persisted on save')
def my_display_name_change_is_persisted_on_save(step):
    world.save_component_and_reopen(step)
    verify_modified_display_name()


@step('the problem display name is "(.*)"$')
def verify_problem_display_name(step, name):
    assert_equal(name.upper(), world.browser.find_by_css('.problem-header').text)


@step('I can specify special characters in the display name')
def i_can_modify_the_display_name_with_special_chars(_step):
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, "updated ' \" &")
    verify_modified_display_name_with_special_chars()


@step('my special characters and persisted on save')
def special_chars_persisted_on_save(step):
    world.save_component_and_reopen(step)
    verify_modified_display_name_with_special_chars()


@step('I can revert the display name to unset')
def can_revert_display_name_to_unset(_step):
    world.revert_setting_entry(DISPLAY_NAME)
    verify_unset_display_name()


@step('my display name is unset on save')
def my_display_name_is_persisted_on_save(step):
    world.save_component_and_reopen(step)
    verify_unset_display_name()


@step('I can select Per Student for Randomization')
def i_can_select_per_student_for_randomization(_step):
    world.browser.select(RANDOMIZATION, "Per Student")
    verify_modified_randomization()


@step('my change to randomization is persisted')
def my_change_to_randomization_is_persisted(step):
    world.save_component_and_reopen(step)
    verify_modified_randomization()


@step('I can revert to the default value for randomization')
def i_can_revert_to_default_for_randomization(step):
    world.revert_setting_entry(RANDOMIZATION)
    world.save_component_and_reopen(step)
    world.verify_setting_entry(world.get_setting_entry(RANDOMIZATION), RANDOMIZATION, "Never", False)


@step('I can set the weight to "(.*)"?')
def i_can_set_weight(_step, weight):
    set_weight(weight)
    verify_modified_weight()


@step('my change to weight is persisted')
def my_change_to_weight_is_persisted(step):
    world.save_component_and_reopen(step)
    verify_modified_weight()


@step('I can revert to the default value of unset for weight')
def i_can_revert_to_default_for_unset_weight(step):
    world.revert_setting_entry(PROBLEM_WEIGHT)
    world.save_component_and_reopen(step)
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "", False)


@step('if I set the weight to "(.*)", it remains unset')
def set_the_weight_to_abc(step, bad_weight):
    set_weight(bad_weight)
    # We show the clear button immediately on type, hence the "True" here.
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "", True)
    world.save_component_and_reopen(step)
    # But no change was actually ever sent to the model, so on reopen, explicitly_set is False
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "", False)


@step('if I set the max attempts to "(.*)", it will persist as a valid integer$')
def set_the_max_attempts(step, max_attempts_set):
    # on firefox with selenium, the behavior is different.
    # eg 2.34 displays as 2.34 and is persisted as 2
    index = world.get_setting_entry_index(MAXIMUM_ATTEMPTS)
    world.set_field_value(index, max_attempts_set)
    world.save_component_and_reopen(step)
    value = world.css_value('input.setting-input', index=index)
    assert value != "", "max attempts is blank"
    assert int(value) >= 0


@step('Edit High Level Source is not visible')
def edit_high_level_source_not_visible(step):
    verify_high_level_source_links(step, False)


@step('Edit High Level Source is visible')
def edit_high_level_source_links_visible(step):
    verify_high_level_source_links(step, True)


@step('If I press Cancel my changes are not persisted')
def cancel_does_not_save_changes(step):
    world.cancel_component(step)
    step.given("I edit and select Settings")
    step.given("I see the advanced settings and their expected values")


@step('I have enabled latex compiler')
def enable_latex_compiler(step):
    url = world.browser.url
    step.given("I select the Advanced Settings")
    change_value(step, 'use_latex_compiler', 'true')
    world.visit(url)
    world.wait_for_xmodule()


@step('I have created a LaTeX Problem')
def create_latex_problem(step):
    world.create_course_with_unit()
    step.given('I have enabled latex compiler')
    world.create_component_instance(
        step=step,
        category='problem',
        component_type='Problem Written in LaTeX',
        is_advanced=True
    )


@step('I edit and compile the High Level Source')
def edit_latex_source(_step):
    open_high_level_source()
    type_in_codemirror(1, "hi")
    world.css_click('.hls-compile')


@step('my change to the High Level Source is persisted')
def high_level_source_persisted(_step):
    def verify_text(driver):
        css_sel = '.problem div>span'
        return world.css_text(css_sel) == 'hi'

    world.wait_for(verify_text, timeout=10)


@step('I view the High Level Source I see my changes')
def high_level_source_in_editor(_step):
    open_high_level_source()
    assert_equal('hi', world.css_value('.source-edit-box'))


@step(u'I have an empty course')
def i_have_empty_course(step):
    open_new_course()


@step(u'I import the file "([^"]*)"$')
def i_import_the_file(_step, filename):
    import_file(filename)


@step(u'I go to the vertical "([^"]*)"$')
def i_go_to_vertical(_step, vertical):
    world.css_click("span:contains('{0}')".format(vertical))


@step(u'I go to the unit "([^"]*)"$')
def i_go_to_unit(_step, unit):
    loc = "window.location = $(\"span:contains('{0}')\").closest('a').attr('href')".format(unit)
    world.browser.execute_script(loc)


@step(u'I see a message that says "([^"]*)"$')
def i_can_see_message(_step, msg):
    msg = json.dumps(msg)     # escape quotes
    world.css_has_text("h2.title", msg)


@step(u'I can edit the problem$')
def i_can_edit_problem(_step):
    world.edit_component()


@step(u'I can see cheatsheet$')
def verify_cheat_sheet_displaying(_step):
    world.css_click("a.cheatsheet-toggle")
    css_selector = 'article.simple-editor-cheatsheet'
    world.wait_for_visible(css_selector)


def verify_high_level_source_links(step, visible):
    if visible:
        assert_true(world.is_css_present('.launch-latex-compiler'),
                    msg="Expected to find the latex button but it is not present.")
    else:
        assert_true(world.is_css_not_present('.launch-latex-compiler'),
                    msg="Expected not to find the latex button but it is present.")

    world.cancel_component(step)


def verify_modified_weight():
    world.verify_setting_entry(world.get_setting_entry(PROBLEM_WEIGHT), PROBLEM_WEIGHT, "3.5", True)


def verify_modified_randomization():
    world.verify_setting_entry(world.get_setting_entry(RANDOMIZATION), RANDOMIZATION, "Per Student", True)


def verify_modified_display_name():
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, '3.4', True)


def verify_modified_display_name_with_special_chars():
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, "updated ' \" &", True)


def verify_unset_display_name():
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, 'Blank Advanced Problem', False)


def set_weight(weight):
    index = world.get_setting_entry_index(PROBLEM_WEIGHT)
    world.set_field_value(index, weight)


def open_high_level_source():
    world.edit_component()
    world.css_click('.launch-latex-compiler > a')

########NEW FILE########
__FILENAME__ = section
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from nose.tools import assert_equal  # pylint: disable=E0611


@step('I click the New Section link$')
def i_click_new_section_link(_step):
    link_css = 'a.new-courseware-section-button'
    world.css_click(link_css)


@step('I enter the section name and click save$')
def i_save_section_name(_step):
    save_section_name('My Section')


@step('I enter a section name with a quote and click save$')
def i_save_section_name_with_quote(_step):
    save_section_name('Section with "Quote"')


@step('I have added a new section$')
def i_have_added_new_section(_step):
    add_section()


@step('I click the Edit link for the release date$')
def i_click_the_edit_link_for_the_release_date(_step):
    button_css = 'div.section-published-date a.edit-release-date'
    world.css_click(button_css)


@step('I set the section release date to ([0-9/-]+)( [0-9:]+)?')
def set_section_release_date(_step, datestring, timestring):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    if not timestring:
        timestring = "00:00"
    set_date_and_time(
        'input.start-date.date.hasDatepicker', datestring,
        'input.start-time.time.ui-timepicker-input', timestring)
    world.browser.click_link_by_text('Save')


@step('I see a "(saving|deleting)" notification')
def i_see_a_mini_notification(_step, _type):
    saving_css = '.wrapper-notification-mini'
    assert world.is_css_present(saving_css)


@step('I see my section on the Courseware page$')
def i_see_my_section_on_the_courseware_page(_step):
    see_my_section_on_the_courseware_page('My Section')


@step('I see my section name with a quote on the Courseware page$')
def i_see_my_section_name_with_quote_on_the_courseware_page(_step):
    see_my_section_on_the_courseware_page('Section with "Quote"')


@step('I click to edit the section name$')
def i_click_to_edit_section_name(_step):
    world.css_click('span.section-name-span')


@step('I see the complete section name with a quote in the editor$')
def i_see_complete_section_name_with_quote_in_editor(_step):
    css = '.section-name-edit input[type=text]'
    assert world.is_css_present(css)
    assert_equal(world.css_value(css), 'Section with "Quote"')


@step('the section does not exist$')
def section_does_not_exist(_step):
    css = 'h3[data-name="My Section"]'
    assert world.is_css_not_present(css)


@step('I see a release date for my section$')
def i_see_a_release_date_for_my_section(_step):
    import re

    css = 'span.published-status'
    assert world.is_css_present(css)
    status_text = world.css_text(css)

    # e.g. 11/06/2012 at 16:25
    msg = 'Release date:'
    date_regex = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d\d?, \d{4}'
    if not re.search(date_regex, status_text):
        print status_text, date_regex
    time_regex = r'[0-2]\d:[0-5]\d( \w{3})?'
    if not re.search(time_regex, status_text):
        print status_text, time_regex
    match_string = r'%s\s+%s at %s' % (msg, date_regex, time_regex)
    if not re.match(match_string, status_text):
        print status_text, match_string
    assert re.match(match_string, status_text)


@step('I see a link to create a new subsection$')
def i_see_a_link_to_create_a_new_subsection(_step):
    css = 'a.new-subsection-item'
    assert world.is_css_present(css)


@step('the section release date picker is not visible$')
def the_section_release_date_picker_not_visible(_step):
    css = 'div.edit-subsection-publish-settings'
    assert not world.css_visible(css)


@step('the section release date is updated$')
def the_section_release_date_is_updated(_step):
    css = 'span.published-status'
    status_text = world.css_text(css)
    assert_equal(status_text, 'Release date: 12/25/2013 at 00:00 UTC')


def save_section_name(name):
    name_css = '.new-section-name'
    save_css = '.new-section-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)


def see_my_section_on_the_courseware_page(name):
    section_css = 'span.section-name-span'
    assert world.css_has_text(section_css, name)

########NEW FILE########
__FILENAME__ = signup
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step


@step('I fill in the registration form$')
def i_fill_in_the_registration_form(step):
    def fill_in_reg_form():
        register_form = world.css_find('form#register_form')
        register_form.find_by_name('email').fill('robot+studio@edx.org')
        register_form.find_by_name('password').fill('test')
        register_form.find_by_name('username').fill('robot-studio')
        register_form.find_by_name('name').fill('Robot Studio')
        register_form.find_by_name('terms_of_service').click()
    world.retry_on_exception(fill_in_reg_form)


@step('I press the Create My Account button on the registration form$')
def i_press_the_button_on_the_registration_form(step):
    submit_css = 'form#register_form button#submit'
    world.css_click(submit_css)


@step('I should see an email verification prompt')
def i_should_see_an_email_verification_prompt(step):
    world.css_has_text('h1.page-header', u'My Courses')
    world.css_has_text('div.msg h3.title', u'We need to verify your email address')


@step(u'I fill in and submit the signin form$')
def i_fill_in_the_signin_form(step):
    def fill_login_form():
        login_form = world.browser.find_by_css('form#login_form')
        login_form.find_by_name('email').fill('robot+studio@edx.org')
        login_form.find_by_name('password').fill('test')
        login_form.find_by_name('submit').click()
    world.retry_on_exception(fill_login_form)

########NEW FILE########
__FILENAME__ = subsection
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from common import *
from nose.tools import assert_equal  # pylint: disable=E0611

############### ACTIONS ####################


@step('I have opened a new course section in Studio$')
def i_have_opened_a_new_course_section(step):
    open_new_course()
    add_section()


@step('I have added a new subsection$')
def i_have_added_a_new_subsection(step):
    add_subsection()


@step('I have opened a new subsection in Studio$')
def i_have_opened_a_new_subsection(step):
    step.given('I have opened a new course section in Studio')
    step.given('I have added a new subsection')
    world.css_click('span.subsection-name-value')


@step('I click the New Subsection link')
def i_click_the_new_subsection_link(step):
    world.css_click('a.new-subsection-item')


@step('I enter the subsection name and click save$')
def i_save_subsection_name(step):
    save_subsection_name('Subsection One')


@step('I enter a subsection name with a quote and click save$')
def i_save_subsection_name_with_quote(step):
    save_subsection_name('Subsection With "Quote"')


@step('I click on the subsection$')
def click_on_subsection(step):
    world.css_click('span.subsection-name-value')


@step('I see the complete subsection name with a quote in the editor$')
def i_see_complete_subsection_name_with_quote_in_editor(step):
    css = '.subsection-display-name-input'
    assert world.is_css_present(css)
    assert_equal(world.css_value(css), 'Subsection With "Quote"')


@step('I set the subsection release date to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_release_date(_step, datestring, timestring):
    set_subsection_date('input#start_date', datestring, 'input#start_time', timestring)


@step('I set the subsection release date on enter to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_release_date_on_enter(_step, datestring, timestring):  # pylint: disable-msg=invalid-name
    set_subsection_date('input#start_date', datestring, 'input#start_time', timestring, 'ENTER')


@step('I set the subsection due date to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_due_date(_step, datestring, timestring, key=None):
    if not world.css_visible('input#due_date'):
        world.css_click('.due-date-input .set-date')

    assert world.css_visible('input#due_date')
    set_subsection_date('input#due_date', datestring, 'input#due_time', timestring, key)


@step('I set the subsection due date on enter to ([0-9/-]+)( [0-9:]+)?')
def set_subsection_due_date_on_enter(_step, datestring, timestring):  # pylint: disable-msg=invalid-name
    set_subsection_due_date(_step, datestring, timestring, 'ENTER')


@step('I mark it as Homework$')
def i_mark_it_as_homework(step):
    world.css_click('a.menu-toggle')
    world.browser.click_link_by_text('Homework')


@step('I see it marked as Homework$')
def i_see_it_marked__as_homework(step):
    assert_equal(world.css_value(".status-label"), 'Homework')


@step('I click the link to sync release date to section')
def click_sync_release_date(step):
    world.css_click('.sync-date')


############ ASSERTIONS ###################


@step('I see my subsection on the Courseware page$')
def i_see_my_subsection_on_the_courseware_page(step):
    see_subsection_name('Subsection One')


@step('I see my subsection name with a quote on the Courseware page$')
def i_see_my_subsection_name_with_quote_on_the_courseware_page(step):
    see_subsection_name('Subsection With "Quote"')


@step('the subsection does not exist$')
def the_subsection_does_not_exist(step):
    css = 'span.subsection-name'
    assert world.is_css_not_present(css)


@step('I see the subsection release date is ([0-9/-]+)( [0-9:]+)?')
def i_see_subsection_release(_step, datestring, timestring):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    assert_equal(datestring, get_date('input#start_date'))
    if timestring:
        assert_equal(timestring, get_date('input#start_time'))


@step('I see the subsection due date is ([0-9/-]+)( [0-9:]+)?')
def i_see_subsection_due(_step, datestring, timestring):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    assert_equal(datestring, get_date('input#due_date'))
    if timestring:
        assert_equal(timestring, get_date('input#due_time'))


############ HELPER METHODS ###################
def get_date(css):
    return world.css_find(css).first.value.strip()


def save_subsection_name(name):
    name_css = 'input.new-subsection-name-input'
    save_css = 'input.new-subsection-name-save'
    world.css_fill(name_css, name)
    world.css_click(save_css)


def see_subsection_name(name):
    css = 'span.subsection-name'
    assert world.is_css_present(css)
    css = 'span.subsection-name-value'
    assert world.css_has_text(css, name)


def set_subsection_date(date_css, datestring, time_css, timestring, key=None):
    if hasattr(timestring, "strip"):
        timestring = timestring.strip()
    if not timestring:
        timestring = "00:00"

    set_date_and_time(date_css, datestring, time_css, timestring, key)

########NEW FILE########
__FILENAME__ = textbooks
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from django.conf import settings
from common import upload_file
from nose.tools import assert_equal

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT


@step(u'I go to the textbooks page')
def go_to_uploads(_step):
    world.wait_for_js_to_load()
    world.click_course_content()
    menu_css = 'li.nav-course-courseware-textbooks a'
    world.css_click(menu_css)


@step(u'I should see a message telling me to create a new textbook')
def assert_create_new_textbook_msg(_step):
    css = ".wrapper-content .no-textbook-content"
    assert world.is_css_present(css)
    no_tb = world.css_find(css)
    assert "You haven't added any textbooks" in no_tb.text


@step(u'I upload the textbook "([^"]*)"$')
def upload_textbook(_step, file_name):
    upload_file(file_name, sub_path="uploads/")


@step(u'I click (on )?the New Textbook button')
def click_new_textbook(_step, on):
    button_css = ".nav-actions .new-button"
    button = world.css_find(button_css)
    button.click()


@step(u'I name my textbook "([^"]*)"')
def name_textbook(_step, name):
    input_css = ".textbook input[name=textbook-name]"
    world.css_fill(input_css, name)
    if world.is_firefox():
        world.trigger_event(input_css)


@step(u'I name the (first|second|third) chapter "([^"]*)"')
def name_chapter(_step, ordinal, name):
    index = ["first", "second", "third"].index(ordinal)
    input_css = ".textbook .chapter{i} input.chapter-name".format(i=index + 1)
    world.css_fill(input_css, name)
    if world.is_firefox():
        world.trigger_event(input_css)


@step(u'I type in "([^"]*)" for the (first|second|third) chapter asset')
def asset_chapter(_step, name, ordinal):
    index = ["first", "second", "third"].index(ordinal)
    input_css = ".textbook .chapter{i} input.chapter-asset-path".format(i=index + 1)
    world.css_fill(input_css, name)
    if world.is_firefox():
        world.trigger_event(input_css)


@step(u'I click the Upload Asset link for the (first|second|third) chapter')
def click_upload_asset(_step, ordinal):
    index = ["first", "second", "third"].index(ordinal)
    button_css = ".textbook .chapter{i} .action-upload".format(i=index + 1)
    world.css_click(button_css)


@step(u'I click Add a Chapter')
def click_add_chapter(_step):
    button_css = ".textbook .action-add-chapter"
    world.css_click(button_css)


@step(u'I save the textbook')
def save_textbook(_step):
    submit_css = "form.edit-textbook button[type=submit]"
    world.css_click(submit_css)


@step(u'I should see a textbook named "([^"]*)" with a chapter path containing "([^"]*)"')
def check_textbook(_step, textbook_name, chapter_name):
    title = world.css_text(".textbook h3.textbook-title", index=0)
    chapter = world.css_text(".textbook .wrap-textbook p", index=0)
    assert_equal(title, textbook_name)
    assert_equal(chapter, chapter_name)


@step(u'I should see a textbook named "([^"]*)" with (\d+) chapters')
def check_textbook_chapters(_step, textbook_name, num_chapters_str):
    num_chapters = int(num_chapters_str)
    title = world.css_text(".textbook .view-textbook h3.textbook-title", index=0)
    toggle_text = world.css_text(".textbook .view-textbook .chapter-toggle", index=0)
    assert_equal(title, textbook_name)
    assert_equal(
        toggle_text,
        "{num} PDF Chapters".format(num=num_chapters),
        "Expected {num} chapters, found {real}".format(num=num_chapters, real=toggle_text)
    )


@step(u'I click the textbook chapters')
def click_chapters(_step):
    world.css_click(".textbook a.chapter-toggle")


@step(u'the (first|second|third) chapter should be named "([^"]*)"')
def check_chapter_name(_step, ordinal, name):
    index = ["first", "second", "third"].index(ordinal)
    chapter = world.css_find(".textbook .view-textbook ol.chapters li")[index]
    element = chapter.find_by_css(".chapter-name")
    assert element.text == name, "Expected chapter named {expected}, found chapter named {actual}".format(
        expected=name, actual=element.text)


@step(u'the (first|second|third) chapter should have an asset called "([^"]*)"')
def check_chapter_asset(_step, ordinal, name):
    index = ["first", "second", "third"].index(ordinal)
    chapter = world.css_find(".textbook .view-textbook ol.chapters li")[index]
    element = chapter.find_by_css(".chapter-asset-path")
    assert element.text == name, "Expected chapter with asset {expected}, found chapter with asset {actual}".format(
        expected=name, actual=element.text)

########NEW FILE########
__FILENAME__ = transcripts
# disable missing docstring
# pylint: disable=C0111

import os
from lettuce import world, step

from django.conf import settings

from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import NotFoundError
from splinter.request_handler.request_handler import RequestHandler

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT

# We should wait 300 ms for event handler invocation + 200ms for safety.
DELAY = 0.5

ERROR_MESSAGES = {
    'url_format': u'Incorrect url format.',
    'file_type': u'Link types should be unique.',
}

STATUSES = {
    'found': u'Timed Transcript Found',
    'not found on edx': u'No EdX Timed Transcript',
    'not found': u'No Timed Transcript',
    'replace': u'Timed Transcript Conflict',
    'uploaded_successfully': u'Timed Transcript Uploaded Successfully',
    'use existing': u'Confirm Timed Transcript',
}

SELECTORS = {
    'error_bar': '.transcripts-error-message',
    'url_inputs': '.videolist-settings-item input.input',
    'collapse_link': '.collapse-action.collapse-setting',
    'collapse_bar': '.videolist-extra-videos',
    'status_bar': '.transcripts-message-status',
}

# button type , button css selector, button message
TRANSCRIPTS_BUTTONS = {
    'import': ('.setting-import', 'Import YouTube Transcript'),
    'download_to_edit': ('.setting-download', 'Download Transcript for Editing'),
    'disabled_download_to_edit': ('.setting-download.is-disabled', 'Download Transcript for Editing'),
    'upload_new_timed_transcripts': ('.setting-upload',  'Upload New Transcript'),
    'replace': ('.setting-replace', 'Yes, replace the edX transcript with the YouTube transcript'),
    'choose': ('.setting-choose', 'Timed Transcript from {}'),
    'use_existing': ('.setting-use-existing', 'Use Current Transcript'),
}


@step('I clear fields$')
def clear_fields(_step):

    # Clear the input fields and trigger an 'input' event
    script = """
        $('{selector}')
            .prop('disabled', false)
            .removeClass('is-disabled')
            .val('')
            .trigger('input');
    """.format(selector=SELECTORS['url_inputs'])
    world.browser.execute_script(script)

    world.wait(DELAY)
    world.wait_for_ajax_complete()


@step('I clear field number (.+)$')
def clear_field(_step, index):
    index = int(index) - 1
    world.css_fill(SELECTORS['url_inputs'], '', index)

    # For some reason ChromeDriver doesn't trigger an 'input' event after filling
    # the field with an empty value. That's why we trigger it manually via jQuery.
    world.trigger_event(SELECTORS['url_inputs'], event='input', index=index)

    world.wait(DELAY)
    world.wait_for_ajax_complete()


@step('I expect (.+) inputs are disabled$')
def inputs_are_disabled(_step, indexes):
    index_list = [int(i.strip()) - 1 for i in indexes.split(',')]
    for index in index_list:
        el = world.css_find(SELECTORS['url_inputs'])[index]

        assert el['disabled']


@step('I expect inputs are enabled$')
def inputs_are_enabled(_step):
    for index in range(3):
        el = world.css_find(SELECTORS['url_inputs'])[index]

        assert not el['disabled']


@step('I do not see error message$')
def i_do_not_see_error_message(_step):
    assert not world.css_visible(SELECTORS['error_bar'])


@step('I see error message "([^"]*)"$')
def i_see_error_message(_step, error):
    assert world.css_has_text(SELECTORS['error_bar'], ERROR_MESSAGES[error])


@step('I do not see status message$')
def i_do_not_see_status_message(_step):
    assert not world.css_visible(SELECTORS['status_bar'])


@step('I see status message "([^"]*)"$')
def i_see_status_message(_step, status):
    assert not world.css_visible(SELECTORS['error_bar'])
    assert world.css_has_text(SELECTORS['status_bar'], STATUSES[status])

    DOWNLOAD_BUTTON = TRANSCRIPTS_BUTTONS["download_to_edit"][0]
    if world.is_css_present(DOWNLOAD_BUTTON, wait_time=1) \
    and not world.css_find(DOWNLOAD_BUTTON)[0].has_class('is-disabled'):
        assert _transcripts_are_downloaded()


@step('I (.*)see button "([^"]*)"$')
def i_see_button(_step, not_see, button_type):
    button = button_type.strip()

    if not_see.strip():
        assert world.is_css_not_present(TRANSCRIPTS_BUTTONS[button][0])
    else:
        assert world.css_has_text(TRANSCRIPTS_BUTTONS[button][0], TRANSCRIPTS_BUTTONS[button][1])


@step('I (.*)see (.*)button "([^"]*)" number (\d+)$')
def i_see_button_with_custom_text(_step, not_see, button_type, custom_text, index):
    button = button_type.strip()
    custom_text = custom_text.strip()
    index = int(index.strip()) - 1

    if not_see.strip():
        assert world.is_css_not_present(TRANSCRIPTS_BUTTONS[button][0])
    else:
        assert world.css_has_text(TRANSCRIPTS_BUTTONS[button][0], TRANSCRIPTS_BUTTONS[button][1].format(custom_text), index)


@step('I click transcript button "([^"]*)"$')
def click_button_transcripts_variant(_step, button_type):
    button = button_type.strip()
    world.css_click(TRANSCRIPTS_BUTTONS[button][0])
    world.wait_for_ajax_complete()


@step('I click transcript button "([^"]*)" number (\d+)$')
def click_button_index(_step, button_type, index):
    button = button_type.strip()
    index = int(index.strip()) - 1

    world.css_click(TRANSCRIPTS_BUTTONS[button][0], index)
    world.wait_for_ajax_complete()


@step('I remove "([^"]+)" transcripts id from store')
def remove_transcripts_from_store(_step, subs_id):
    """Remove from store, if transcripts content exists."""
    filename = 'subs_{0}.srt.sjson'.format(subs_id.strip())
    content_location = StaticContent.compute_location(
        world.scenario_dict['COURSE'].id,
        filename
    )
    try:
        content = contentstore().find(content_location)
        contentstore().delete(content.get_id())
        print('Transcript file was removed from store.')
    except NotFoundError:
        print('Transcript file was NOT found and not removed.')


@step('I enter a "([^"]+)" source to field number (\d+)$')
def i_enter_a_source(_step, link, index):
    index = int(index) - 1

    if index is not 0 and not world.css_visible(SELECTORS['collapse_bar']):
        world.css_click(SELECTORS['collapse_link'])

        assert world.css_visible(SELECTORS['collapse_bar'])

    world.css_fill(SELECTORS['url_inputs'], link, index)
    world.wait(DELAY)
    world.wait_for_ajax_complete()


@step('I upload the transcripts file "([^"]*)"$')
def upload_file(_step, file_name):
    path = os.path.join(TEST_ROOT, 'uploads/', file_name.strip())
    world.browser.execute_script("$('form.file-chooser').show()")
    world.browser.attach_file('transcript-file', os.path.abspath(path))
    world.wait_for_ajax_complete()


@step('I see "([^"]*)" text in the captions')
def check_text_in_the_captions(_step, text):
    world.wait_for_present('.video.is-captions-rendered')
    world.wait_for(lambda _: world.css_text('.subtitles'), timeout=30)
    actual_text = world.css_text('.subtitles')
    assert (text in actual_text)


@step('I see value "([^"]*)" in the field "([^"]*)"$')
def check_transcripts_field(_step, values, field_name):
    world.select_editor_tab('Advanced')
    tab = world.css_find('#settings-tab').first;
    field_id = '#' + tab.find_by_xpath('.//label[text()="%s"]' % field_name.strip())[0]['for']
    values_list = [i.strip() == world.css_value(field_id) for i in values.split('|')]
    assert any(values_list)
    world.select_editor_tab('Basic')


@step('I save changes$')
def save_changes(_step):
    world.save_component()


@step('I open tab "([^"]*)"$')
def open_tab(_step, tab_name):
    world.select_editor_tab(tab_name)


@step('I set value "([^"]*)" to the field "([^"]*)"$')
def set_value_transcripts_field(_step, value, field_name):
    tab = world.css_find('#settings-tab').first;
    XPATH = './/label[text()="{name}"]'.format(name=field_name)
    SELECTOR = '#' + tab.find_by_xpath(XPATH)[0]['for']
    element = world.css_find(SELECTOR).first
    if element['type'] == 'text':
        SCRIPT = '$("{selector}").val("{value}").change()'.format(
                selector=SELECTOR,
                value=value
            )
        world.browser.execute_script(SCRIPT)
        assert world.css_has_value(SELECTOR, value)
    else:
        assert False, 'Incorrect element type.';
    world.wait_for_ajax_complete()


@step('I revert the transcript field "([^"]*)"$')
def revert_transcripts_field(_step, field_name):
    world.revert_setting_entry(field_name)


def _transcripts_are_downloaded():
    world.wait_for_ajax_complete()
    request = RequestHandler()
    DOWNLOAD_BUTTON = world.css_find(TRANSCRIPTS_BUTTONS["download_to_edit"][0]).first
    url = DOWNLOAD_BUTTON['href']
    request.connect(url)

    return request.status_code.is_success()

########NEW FILE########
__FILENAME__ = upload
# pylint: disable=C0111
# pylint: disable=W0621

from lettuce import world, step
from lettuce.django import django_url
from django.conf import settings
import requests
import string
import random
import os
from django.contrib.auth.models import User
from student.models import CourseEnrollment
from nose.tools import assert_equal, assert_not_equal  # pylint: disable=E0611

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT
ASSET_NAMES_CSS = 'td.name-col > span.title > a.filename'


@step(u'I go to the files and uploads page$')
def go_to_uploads(_step):
    menu_css = 'li.nav-course-courseware'
    uploads_css = 'li.nav-course-courseware-uploads a'
    world.css_click(menu_css)
    world.css_click(uploads_css)


@step(u'I upload the( test)? file "([^"]*)"$')
def upload_file(_step, is_test_file, file_name, button_text=None):
    if button_text:
        world.click_link(button_text)
    else:
        world.click_link('Upload New File')

    if not is_test_file:
        _write_test_file(file_name, "test file")

    # uploading the file itself
    path = os.path.join(TEST_ROOT, 'uploads/', file_name)
    world.browser.execute_script("$('input.file-input').css('display', 'block')")
    world.browser.attach_file('file', os.path.abspath(path))
    close_css = 'a.close-button'
    world.css_click(close_css)


@step(u'I upload the file "([^"]*)" by clicking "([^"]*)"')
def upload_file_on_button_press(_step, file_name, button_text=None):
    upload_file(_step, '', file_name, button_text)


@step(u'I upload the files "([^"]*)"$')
def upload_files(_step, files_string):
    # files_string should be comma separated with no spaces.
    files = files_string.split(",")
    upload_css = 'a.upload-button'
    world.css_click(upload_css)

    # uploading the files
    for filename in files:
        _write_test_file(filename, "test file")
        path = os.path.join(TEST_ROOT, 'uploads/', filename)
        world.browser.execute_script("$('input.file-input').css('display', 'block')")
        world.browser.attach_file('file', os.path.abspath(path))

    close_css = 'a.close-button'
    world.css_click(close_css)


@step(u'I should not see the file "([^"]*)" was uploaded$')
def check_not_there(_step, file_name):
    # Either there are no files, or there are files but
    # not the one I expect not to exist.

    # Since our only test for deletion right now deletes
    # the only file that was uploaded, our success criteria
    # will be that there are no files.
    # In the future we can refactor if necessary.
    assert(world.is_css_not_present(ASSET_NAMES_CSS))


@step(u'I should see the file "([^"]*)" was uploaded$')
def check_upload(_step, file_name):
    index = get_index(file_name)
    assert_not_equal(index, -1)


@step(u'The url for the file "([^"]*)" is valid$')
def check_url(_step, file_name):
    r = get_file(file_name)
    assert_equal(r.status_code, 200)


@step(u'I delete the file "([^"]*)"$')
def delete_file(_step, file_name):
    index = get_index(file_name)
    assert index != -1
    delete_css = "a.remove-asset-button"
    world.css_click(delete_css, index=index)
    world.confirm_studio_prompt()


@step(u'I should see only one "([^"]*)"$')
def no_duplicate(_step, file_name):
    all_names = world.css_find(ASSET_NAMES_CSS)
    only_one = False
    for i in range(len(all_names)):
        if file_name == world.css_html(ASSET_NAMES_CSS, index=i):
            only_one = not only_one
    assert only_one


@step(u'I can download the correct "([^"]*)" file$')
def check_download(_step, file_name):
    path = os.path.join(TEST_ROOT, 'uploads/', file_name)
    with open(os.path.abspath(path), 'r') as cur_file:
        cur_text = cur_file.read()
        r = get_file(file_name)
        downloaded_text = r.text
        assert cur_text == downloaded_text
    # resetting the file back to its original state
    _write_test_file(file_name, "This is an arbitrary file for testing uploads")


def _write_test_file(file_name, text):
    path = os.path.join(TEST_ROOT, 'uploads/', file_name)
    # resetting the file back to its original state
    with open(os.path.abspath(path), 'w') as cur_file:
        cur_file.write(text)


@step(u'I modify "([^"]*)"$')
def modify_upload(_step, file_name):
    new_text = ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(10))
    _write_test_file(file_name, new_text)


@step(u'I upload an asset$')
def upload_an_asset(step):
    step.given('I upload the file "asset.html"')


@step(u'I (lock|unlock) the asset$')
def lock_unlock_file(_step, _lock_state):
    index = get_index('asset.html')
    assert index != -1, 'Expected to find an asset but could not.'

    # Warning: this is a misnomer, it really only toggles the
    # lock state. TODO: fix it.
    lock_css = "input.lock-checkbox"
    world.css_find(lock_css)[index].click()


@step(u'the user "([^"]*)" is enrolled in the course$')
def user_foo_is_enrolled_in_the_course(step, name):
    world.create_user(name, 'test')
    user = User.objects.get(username=name)

    course_id = world.scenario_dict['COURSE'].id
    CourseEnrollment.enroll(user, course_id)


@step(u'Then the asset is (locked|unlocked)$')
def verify_lock_unlock_file(_step, lock_state):
    index = get_index('asset.html')
    assert index != -1, 'Expected to find an asset but could not.'
    lock_css = "input.lock-checkbox"
    checked = world.css_find(lock_css)[index]._element.get_attribute('checked')
    assert_equal(lock_state == "locked", bool(checked))


@step(u'I am at the files and upload page of a Studio course')
def at_upload_page(step):
    step.given('I have opened a new course in studio')
    step.given('I go to the files and uploads page')


@step(u'I have created a course with a (locked|unlocked) asset$')
def open_course_with_locked(step, lock_state):
    step.given('I am at the files and upload page of a Studio course')
    step.given('I upload the file "asset.html"')

    if lock_state == "locked":
        step.given('I lock the asset')
        step.given('I reload the page')


@step(u'Then the asset is (viewable|protected)$')
def view_asset(_step, status):
    url = django_url('/c4x/MITx/999/asset/asset.html')
    if status == 'viewable':
        expected_text = 'test file'
    else:
        expected_text = 'Unauthorized'

    # Note that world.visit would trigger a 403 error instead of displaying "Unauthorized"
    # Instead, we can drop back into the selenium driver get command.
    world.browser.driver.get(url)
    assert_equal(world.css_text('body'), expected_text)


@step('I see a confirmation that the file was deleted$')
def i_see_a_delete_confirmation(_step):
    alert_css = '#notification-confirmation'
    assert world.is_css_present(alert_css)


def get_index(file_name):
    all_names = world.css_find(ASSET_NAMES_CSS)
    for i in range(len(all_names)):
        if file_name == world.css_html(ASSET_NAMES_CSS, index=i):
            return i
    return -1


def get_file(file_name):
    index = get_index(file_name)
    assert index != -1
    url_css = 'a.filename'

    def get_url():
        return world.css_find(url_css)[index]._element.get_attribute('href')
    url = world.retry_on_exception(get_url)
    return requests.get(url)

########NEW FILE########
__FILENAME__ = video
# pylint: disable=C0111

from lettuce import world, step
from nose.tools import assert_less
from xmodule.modulestore import Location
from contentstore.utils import get_modulestore
from selenium.webdriver.common.keys import Keys

VIDEO_BUTTONS = {
    'CC': '.hide-subtitles',
    'volume': '.volume',
    'play': '.video_control.play',
    'pause': '.video_control.pause',
    'handout': '.video-handout.video-download-button a',
}

SELECTORS = {
    'spinner': '.video-wrapper .spinner',
    'controls': 'section.video-controls',
}

# We should wait 300 ms for event handler invocation + 200ms for safety.
DELAY = 0.5


@step('youtube stub server (.*) YouTube API')
def configure_youtube_api(_step, action):
    action=action.strip()
    if action == 'proxies':
        world.youtube.config['youtube_api_blocked'] = False
    elif action == 'blocks':
        world.youtube.config['youtube_api_blocked'] = True
    else:
        raise ValueError('Parameter `action` should be one of "proxies" or "blocks".')


@step('I have created a Video component$')
def i_created_a_video_component(_step):

    world.create_course_with_unit()
    world.create_component_instance(
        step=_step,
        category='video',
    )

    world.wait_for_xmodule()
    world.disable_jquery_animations()

    world.wait_for_present('.is-initialized')
    world.wait(DELAY)
    world.wait_for_invisible(SELECTORS['spinner'])
    if not world.youtube.config.get('youtube_api_blocked'):
        world.wait_for_visible(SELECTORS['controls'])

@step('I have created a Video component with subtitles$')
def i_created_a_video_with_subs(_step):
    _step.given('I have created a Video component with subtitles "OEoXaMPEzfM"')


@step('I have created a Video component with subtitles "([^"]*)"$')
def i_created_a_video_with_subs_with_name(_step, sub_id):
    _step.given('I have created a Video component')

    # Store the current URL so we can return here
    video_url = world.browser.url

    # Upload subtitles for the video using the upload interface
    _step.given('I have uploaded subtitles "{}"'.format(sub_id))

    # Return to the video
    world.visit(video_url)

    world.wait_for_xmodule()

    # update .sub filed with proper subs name (which mimics real Studio/XML behavior)
    # this is needed only for that videos which are created in acceptance tests.
    _step.given('I edit the component')
    world.wait_for_ajax_complete()
    _step.given('I save changes')

    world.disable_jquery_animations()

    world.wait_for_present('.is-initialized')
    world.wait_for_invisible(SELECTORS['spinner'])


@step('I have uploaded subtitles "([^"]*)"$')
def i_have_uploaded_subtitles(_step, sub_id):
    _step.given('I go to the files and uploads page')
    _step.given('I upload the test file "subs_{}.srt.sjson"'.format(sub_id.strip()))


@step('when I view the (.*) it does not have autoplay enabled$')
def does_not_autoplay(_step, video_type):
    world.wait(DELAY)
    world.wait_for_ajax_complete()
    actual = world.css_find('.%s' % video_type)[0]['data-autoplay']
    expected = [u'False', u'false', False]
    assert actual in expected
    assert world.css_has_class('.video_control', 'play')


@step('creating a video takes a single click$')
def video_takes_a_single_click(_step):
    component_css = '.xmodule_VideoModule'
    assert world.is_css_not_present(component_css)

    world.css_click("a[data-category='video']")
    assert world.is_css_present(component_css)


@step('I edit the component$')
def i_edit_the_component(_step):
    world.edit_component()


@step('I have (hidden|toggled) captions$')
def hide_or_show_captions(step, shown):
    button_css = 'a.hide-subtitles'
    if shown == 'hidden':
        world.css_click(button_css)
    if shown == 'toggled':
        world.css_click(button_css)
        # When we click the first time, a tooltip shows up. We want to
        # click the button rather than the tooltip, so move the mouse
        # away to make it disappear.
        button = world.css_find(button_css)
        # mouse_out is not implemented on firefox with selenium
        if not world.is_firefox:
            button.mouse_out()
        world.css_click(button_css)


@step('I have created a video with only XML data$')
def xml_only_video(step):
    # Create a new video *without* metadata. This requires a certain
    # amount of rummaging to make sure all the correct data is present
    step.given('I have clicked the new unit button')

    # Wait for the new unit to be created and to load the page
    world.wait(1)

    course = world.scenario_dict['COURSE']
    store = get_modulestore(course.location)

    parent_location = store.get_items(course.id, category='vertical', revision='draft')[0].location

    youtube_id = 'ABCDEFG'
    world.scenario_dict['YOUTUBE_ID'] = youtube_id

    # Create a new Video component, but ensure that it doesn't have
    # metadata. This allows us to test that we are correctly parsing
    # out XML
    world.ItemFactory.create(
        parent_location=parent_location,
        category='video',
        data='<video youtube="1.00:%s"></video>' % youtube_id,
        modulestore=store,
    )


@step('The correct Youtube video is shown$')
def the_youtube_video_is_shown(_step):
    ele = world.css_find('.video').first
    assert ele['data-streams'].split(':')[1] == world.scenario_dict['YOUTUBE_ID']


@step('Make sure captions are (.+)$')
def set_captions_visibility_state(_step, captions_state):
    SELECTOR = '.closed .subtitles'
    world.wait_for_visible('.hide-subtitles')
    if captions_state == 'closed':
        if not world.is_css_present(SELECTOR):
            world.css_find('.hide-subtitles').click()
    else:
        if world.is_css_present(SELECTOR):
            world.css_find('.hide-subtitles').click()


@step('I hover over button "([^"]*)"$')
def hover_over_button(_step, button):
    world.css_find(VIDEO_BUTTONS[button.strip()]).mouse_over()


@step('Captions (?:are|become) "([^"]*)"$')
def check_captions_visibility_state(_step, visibility_state):
    if visibility_state == 'visible':
        assert world.css_visible('.subtitles')
    else:
        assert not world.css_visible('.subtitles')


def find_caption_line_by_data_index(index):
    SELECTOR = ".subtitles > li[data-index='{index}']".format(index=index)
    return world.css_find(SELECTOR).first


@step('I focus on caption line with data-index "([^"]*)"$')
def focus_on_caption_line(_step, index):
    world.wait_for_present('.video.is-captions-rendered')
    world.wait_for(lambda _: world.css_text('.subtitles'), timeout=30)
    find_caption_line_by_data_index(int(index.strip()))._element.send_keys(Keys.TAB)


@step('I press "enter" button on caption line with data-index "([^"]*)"$')
def click_on_the_caption(_step, index):
    world.wait_for_present('.video.is-captions-rendered')
    world.wait_for(lambda _: world.css_text('.subtitles'), timeout=30)
    find_caption_line_by_data_index(int(index.strip()))._element.send_keys(Keys.ENTER)


@step('I see caption line with data-index "([^"]*)" has class "([^"]*)"$')
def caption_line_has_class(_step, index, className):
    SELECTOR = ".subtitles > li[data-index='{index}']".format(index=int(index.strip()))
    assert world.css_has_class(SELECTOR, className.strip())


@step('I see a range on slider$')
def see_a_range_slider_with_proper_range(_step):
    world.wait_for_visible(VIDEO_BUTTONS['pause'])
    assert world.css_visible(".slider-range")


@step('I (.*) see video button "([^"]*)"$')
def do_not_see_or_not_button_video(_step, action, button_type):
    world.wait(DELAY)
    world.wait_for_ajax_complete()
    action=action.strip()
    button = button_type.strip()
    if action == 'do not':
        assert not world.is_css_present(VIDEO_BUTTONS[button])
    elif action == 'can':
        assert world.css_visible(VIDEO_BUTTONS[button])
    else:
        raise ValueError('Parameter `action` should be one of "do not" or "can".')


@step('I click video button "([^"]*)"$')
def click_button_video(_step, button_type):
    world.wait(DELAY)
    world.wait_for_ajax_complete()
    button = button_type.strip()
    world.css_click(VIDEO_BUTTONS[button])


@step('I seek video to "([^"]*)" seconds$')
def seek_video_to_n_seconds(_step, seconds):
    time = float(seconds.strip())
    jsCode = "$('.video').data('video-player-state').videoPlayer.onSlideSeek({{time: {0:f}}})".format(time)
    world.browser.execute_script(jsCode)


@step('I see video starts playing from "([^"]*)" position$')
def start_playing_video_from_n_seconds(_step, position):
    world.wait_for(
        func=lambda _: world.css_html('.vidtime')[:4] == position.strip(),
        timeout=5
    )

########NEW FILE########
__FILENAME__ = video_editor
# -*- coding: utf-8 -*-
# disable missing docstring
# pylint: disable=C0111

import requests
from lettuce import world, step
from nose.tools import assert_true, assert_equal, assert_in, assert_not_equal # pylint: disable=E0611
from terrain.steps import reload_the_page
from django.conf import settings
from common import upload_file, attach_file

TEST_ROOT = settings.COMMON_TEST_DATA_ROOT

DISPLAY_NAME = "Component Display Name"
NATIVE_LANGUAGES = {lang: label for lang, label in settings.LANGUAGES if len(lang) == 2}
LANGUAGES = {
    lang: NATIVE_LANGUAGES.get(lang, display)
    for lang, display in settings.ALL_LANGUAGES
}

TRANSLATION_BUTTONS = {
    'add': '.metadata-video-translations .create-action',
    'upload': '.metadata-video-translations .upload-action',
    'download': '.metadata-video-translations .download-action',
    'remove': '.metadata-video-translations .remove-action',
    'clear': '.metadata-video-translations .setting-clear',
}

VIDEO_MENUS = {
    'language': '.lang .menu',
}


class RequestHandlerWithSessionId(object):
    def get(self, url):
        """
        Sends a request.
        """
        kwargs = dict()

        session_id = [{i['name']:i['value']} for i in world.browser.cookies.all() if i['name'] == u'sessionid']
        if session_id:
            kwargs.update({
                'cookies': session_id[0]
            })

        response = requests.get(url, **kwargs)
        self.response = response
        self.status_code = response.status_code
        self.headers = response.headers
        self.content = response.content

        return self

    def is_success(self):
        """
        Returns `True` if the response was succeed, otherwise, returns `False`.
        """
        if self.status_code < 400:
            return True
        return False

    def check_header(self, name, value):
        """
        Returns `True` if the response header exist and has appropriate value,
        otherwise, returns `False`.
        """
        if value in self.headers.get(name, ''):
            return True
        return False


def success_upload_file(filename):
    upload_file(filename, sub_path="uploads/")
    world.css_has_text('#upload_confirm', 'Success!')
    world.is_css_not_present('.wrapper-modal-window-assetupload', wait_time=30)


def get_translations_container():
    return world.browser.find_by_xpath('//label[text()="Transcript Languages"]/following-sibling::div')


def get_setting_container(lang_code):
    try:
        get_xpath = lambda value: './/descendant::a[@data-lang="{}" and contains(@class,"remove-setting")]/parent::*'.format(value)
        return get_translations_container().find_by_xpath(get_xpath(lang_code)).first
    except Exception:
        return None


def get_last_dropdown():
    return get_translations_container().find_by_xpath('.//descendant::select[last()]').last


def choose_option(dropdown, value):
    dropdown.find_by_value(value)[0].click()


def choose_new_lang(lang_code):
    world.css_click(TRANSLATION_BUTTONS['add'])
    choose_option(get_last_dropdown(), lang_code)
    assert_equal(get_last_dropdown().value, lang_code, "Option with provided value is not available or was not selected")


def open_menu(menu):
    world.browser.execute_script("$('{selector}').parent().addClass('is-opened')".format(
        selector=VIDEO_MENUS[menu]
    ))


@step('I have set "transcript display" to (.*)$')
def set_show_captions(step, setting):
    # Prevent cookies from overriding course settings
    world.browser.cookies.delete('hide_captions')

    world.edit_component()
    world.select_editor_tab('Advanced')
    world.browser.select('Show Transcript', setting)
    world.save_component()


@step('when I view the video it (.*) show the captions$')
def shows_captions(_step, show_captions):
    world.wait_for_js_variable_truthy("Video")
    world.wait(0.5)
    if show_captions == 'does not':
        assert_true(world.is_css_present('div.video.closed'))
    else:
        assert_true(world.is_css_not_present('div.video.closed'))

    # Prevent cookies from overriding course settings
    world.browser.cookies.delete('hide_captions')
    world.browser.cookies.delete('current_player_mode')


@step('I see the correct video settings and default values$')
def correct_video_settings(_step):
    expected_entries = [
        # basic
        [DISPLAY_NAME, 'Video', False],
        ['Default Video URL', 'http://youtu.be/OEoXaMPEzfM, , ', False],

        # advanced
        [DISPLAY_NAME, 'Video', False],
        ['Default Timed Transcript', '', False],
        ['Download Transcript Allowed', 'False', False],
        ['Downloadable Transcript URL', '', False],
        ['Show Transcript', 'True', False],
        ['Transcript Languages', '', False],
        ['Upload Handout', '', False],
        ['Video Download Allowed', 'False', False],
        ['Video File URLs', '', False],
        ['Video Start Time', '00:00:00', False],
        ['Video Stop Time', '00:00:00', False],
        ['YouTube ID', 'OEoXaMPEzfM', False],
        ['YouTube ID for .75x speed', '', False],
        ['YouTube ID for 1.25x speed', '', False],
        ['YouTube ID for 1.5x speed', '', False]
    ]
    world.verify_all_setting_entries(expected_entries)


@step('my video display name change is persisted on save$')
def video_name_persisted(step):
    world.save_component()
    reload_the_page(step)
    world.wait_for_xmodule()
    world.edit_component()

    world.verify_setting_entry(
        world.get_setting_entry(DISPLAY_NAME),
        DISPLAY_NAME, '3.4', True
    )


@step('I can modify video display name')
def i_can_modify_video_display_name(_step):
    index = world.get_setting_entry_index(DISPLAY_NAME)
    world.set_field_value(index, '3.4')
    world.verify_setting_entry(world.get_setting_entry(DISPLAY_NAME), DISPLAY_NAME, '3.4', True)


@step('I upload transcript file(?:s)?:$')
def upload_transcript(step):
    input_hidden = '.metadata-video-translations .input'
    # Number of previously added translations
    initial_index = len(world.css_find(TRANSLATION_BUTTONS['download']))

    if step.hashes:
        for i, item in enumerate(step.hashes):
            lang_code = item['lang_code']
            filename = item['filename']
            index = initial_index + i

            choose_new_lang(lang_code)

            expected_text = world.css_text(TRANSLATION_BUTTONS['upload'], index=index)
            assert_equal(expected_text, "Upload")
            assert_equal(world.css_find(input_hidden).last.value, "")

            world.css_click(TRANSLATION_BUTTONS['upload'], index=index)
            success_upload_file(filename)

            world.wait_for_visible(TRANSLATION_BUTTONS['download'], index=index)
            assert_equal(world.css_find(TRANSLATION_BUTTONS['upload']).last.text, "Replace")
            assert_equal(world.css_find(input_hidden).last.value, filename)


@step('I try to upload transcript file "([^"]*)"$')
def try_to_upload_transcript(step, filename):
    world.css_click(TRANSLATION_BUTTONS['upload'])
    attach_file(filename, 'uploads/')


@step('I upload transcript file "([^"]*)" for "([^"]*)" language code$')
def upload_transcript_for_lang(step, filename, lang_code):
    get_xpath = lambda value: './/div/a[contains(@class, "upload-action")]'.format(value)
    container = get_setting_container(lang_code)

    # If translation isn't uploaded, prepare drop-down and try to find container again
    choose_new_lang(lang_code)
    container = get_setting_container(lang_code)

    button = container.find_by_xpath(get_xpath(lang_code)).first
    button.click()
    success_upload_file(filename)


@step('I replace transcript file for "([^"]*)" language code by "([^"]*)"$')
def replace_transcript_for_lang(step, lang_code, filename):
    get_xpath = lambda value: './/div/a[contains(@class, "upload-action")]'.format(value)
    container = get_setting_container(lang_code)

    button = container.find_by_xpath(get_xpath(lang_code)).first
    button.click()
    success_upload_file(filename)


@step('I see validation error "([^"]*)"$')
def verify_validation_error_message(step, error_message):
    assert_equal(world.css_text('#upload_error'), error_message)


@step('I can download transcript for "([^"]*)" language code, that contains text "([^"]*)"$')
def i_can_download_transcript(_step, lang_code, text):
    MIME_TYPE = 'application/x-subrip'
    get_xpath = lambda value: './/div/a[contains(text(), "Download")]'.format(value)
    container = get_setting_container(lang_code)
    assert container
    button = container.find_by_xpath(get_xpath(lang_code)).first
    url = button['href']
    request = RequestHandlerWithSessionId()
    assert_true(request.get(url).is_success())
    assert_true(request.check_header('content-type', MIME_TYPE))
    assert_in(text.encode('utf-8'), request.content)


@step('I remove translation for "([^"]*)" language code$')
def i_can_remove_transcript(_step, lang_code):
    get_xpath = lambda value: './/descendant::a[@data-lang="{}" and contains(@class,"remove-setting")]'.format(value)
    container = get_setting_container(lang_code)
    assert container
    button = container.find_by_xpath(get_xpath(lang_code)).first
    button.click()


@step('I see translations for "([^"]*)"$')
def verify_translations(_step, lang_codes_string):
    expected = [l.strip() for l in lang_codes_string.split(',')]
    actual = [l['data-lang'] for l in world.css_find('.metadata-video-translations .remove-setting')]
    assert_equal(set(expected), set(actual))


@step('I do not see translations$')
def no_translations(_step):
    assert_true(world.is_css_not_present('.metadata-video-translations .remove-setting'))


@step('I confirm prompt$')
def confirm_prompt(_step):
    world.confirm_studio_prompt()


@step('I (cannot )?choose "([^"]*)" language code$')
def i_choose_lang_code(_step, cannot, lang_code):
    choose_option(get_last_dropdown(), lang_code)
    if cannot:
        assert_not_equal(get_last_dropdown().value, lang_code, "Option with provided value was selected, but shouldn't")
    else:
        assert_equal(get_last_dropdown().value, lang_code, "Option with provided value is not available or was not selected")


@step('I click button "([^"]*)"$')
def click_button(_step, button):
    world.css_click(TRANSLATION_BUTTONS[button.lower()])


@step('video language menu has "([^"]*)" translations$')
def i_see_correct_langs(_step, langs):
    menu_name = 'language'
    open_menu(menu_name)
    items = world.css_find(VIDEO_MENUS[menu_name] + ' li')
    translations = {t.strip(): LANGUAGES[t.strip()] for t in langs.split(',')}

    assert_equal(len(translations), len(items))
    for lang_code, label in translations.items():
        assert_true(any([i.text == label for i in items]))
        assert_true(any([i['data-lang-code'] == lang_code for i in items]))

########NEW FILE########
__FILENAME__ = video_handout
# -*- coding: utf-8 -*-
# disable missing docstring
# pylint: disable=C0111

from lettuce import world, step
from nose.tools import assert_true # pylint: disable=E0611
from video_editor import RequestHandlerWithSessionId, success_upload_file


@step('I (?:upload|replace) handout file(?: by)? "([^"]*)"$')
def upload_handout(step, filename):
    world.css_click('.wrapper-comp-setting.file-uploader .upload-action')
    success_upload_file(filename)


@step('I can download handout file( in editor)? with mime type "([^"]*)"$')
def i_can_download_handout_with_mime_type(_step, is_editor, mime_type):
    if is_editor:
        selector = '.wrapper-comp-setting.file-uploader .download-action'
    else:
        selector = '.video-handout.video-download-button a'

    button = world.css_find(selector).first
    url = button['href']
    request = RequestHandlerWithSessionId()
    assert_true(request.get(url).is_success())
    assert_true(request.check_header('content-type', mime_type))


@step('I clear handout$')
def clear_handout(_step):
    world.css_click('.wrapper-comp-setting.file-uploader .setting-clear')


@step('I have created a Video component with handout file "([^"]*)"')
def create_video_with_handout(_step, filename):
    _step.given('I have created a Video component')
    _step.given('I edit the component')
    _step.given('I open tab "Advanced"')
    _step.given('I upload handout file "{0}"'.format(filename))

########NEW FILE########
__FILENAME__ = git_export_utils
"""
Utilities for export a course's XML into a git repository,
committing and pushing the changes.
"""

import logging
import os
import subprocess
from urlparse import urlparse

from django.conf import settings
from django.contrib.auth.models import User
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _

from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_exporter import export_to_xml

log = logging.getLogger(__name__)

GIT_REPO_EXPORT_DIR = getattr(settings, 'GIT_REPO_EXPORT_DIR', None)
GIT_EXPORT_DEFAULT_IDENT = getattr(settings, 'GIT_EXPORT_DEFAULT_IDENT',
                                   {'name': 'STUDIO_EXPORT_TO_GIT',
                                    'email': 'STUDIO_EXPORT_TO_GIT@example.com'})


class GitExportError(Exception):
    """
    Convenience exception class for git export error conditions.
    """

    NO_EXPORT_DIR = _("GIT_REPO_EXPORT_DIR not set or path {0} doesn't exist, "
                      "please create it, or configure a different path with "
                      "GIT_REPO_EXPORT_DIR".format(GIT_REPO_EXPORT_DIR))
    URL_BAD = _('Non writable git url provided. Expecting something like:'
                ' git@github.com:mitocw/edx4edx_lite.git')
    URL_NO_AUTH = _('If using http urls, you must provide the username '
                    'and password in the url. Similar to '
                    'https://user:pass@github.com/user/course.')
    DETACHED_HEAD = _('Unable to determine branch, repo in detached HEAD mode')
    CANNOT_PULL = _('Unable to update or clone git repository.')
    XML_EXPORT_FAIL = _('Unable to export course to xml.')
    CONFIG_ERROR = _('Unable to configure git username and password')
    CANNOT_COMMIT = _('Unable to commit changes. This is usually '
                      'because there are no changes to be committed')
    CANNOT_PUSH = _('Unable to push changes.  This is usually '
                    'because the remote repository cannot be contacted')
    BAD_COURSE = _('Bad course location provided')
    MISSING_BRANCH = _('Missing branch on fresh clone')


def cmd_log(cmd, cwd):
    """
    Helper function to redirect stderr to stdout and log the command
    used along with the output. Will raise subprocess.CalledProcessError if
    command doesn't return 0, and returns the command's output.
    """
    output = subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT)
    log.debug(_('Command was: {0!r}. '
                'Working directory was: {1!r}'.format(' '.join(cmd), cwd)))
    log.debug(_('Command output was: {0!r}'.format(output)))
    return output


def export_to_git(course_id, repo, user='', rdir=None):
    """Export a course to git."""
    # pylint: disable=R0915

    if not GIT_REPO_EXPORT_DIR:
        raise GitExportError(GitExportError.NO_EXPORT_DIR)

    if not os.path.isdir(GIT_REPO_EXPORT_DIR):
        raise GitExportError(GitExportError.NO_EXPORT_DIR)

    # Check for valid writable git url
    if not (repo.endswith('.git') or
            repo.startswith(('http:', 'https:', 'file:'))):
        raise GitExportError(GitExportError.URL_BAD)

    # Check for username and password if using http[s]
    if repo.startswith('http:') or repo.startswith('https:'):
        parsed = urlparse(repo)
        if parsed.username is None or parsed.password is None:
            raise GitExportError(GitExportError.URL_NO_AUTH)
    if rdir:
        rdir = os.path.basename(rdir)
    else:
        rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]

    log.debug("rdir = %s", rdir)

    # Pull or clone repo before exporting to xml
    # and update url in case origin changed.
    rdirp = '{0}/{1}'.format(GIT_REPO_EXPORT_DIR, rdir)
    branch = None
    if os.path.exists(rdirp):
        log.info(_('Directory already exists, doing a git reset and pull '
                   'instead of git clone.'))
        cwd = rdirp
        # Get current branch
        cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
        try:
            branch = cmd_log(cmd, cwd).strip('\n')
        except subprocess.CalledProcessError as ex:
            log.exception('Failed to get branch: %r', ex.output)
            raise GitExportError(GitExportError.DETACHED_HEAD)

        cmds = [
            ['git', 'remote', 'set-url', 'origin', repo],
            ['git', 'fetch', 'origin'],
            ['git', 'reset', '--hard', 'origin/{0}'.format(branch)],
            ['git', 'pull'],
        ]
    else:
        cmds = [['git', 'clone', repo]]
        cwd = GIT_REPO_EXPORT_DIR

    cwd = os.path.abspath(cwd)
    for cmd in cmds:
        try:
            cmd_log(cmd, cwd)
        except subprocess.CalledProcessError as ex:
            log.exception('Failed to pull git repository: %r', ex.output)
            raise GitExportError(GitExportError.CANNOT_PULL)

    # export course as xml before commiting and pushing
    root_dir = os.path.dirname(rdirp)
    course_dir = os.path.splitext(os.path.basename(rdirp))[0]
    try:
        export_to_xml(modulestore('direct'), contentstore(), course_id,
                      root_dir, course_dir, modulestore())
    except (EnvironmentError, AttributeError):
        log.exception('Failed export to xml')
        raise GitExportError(GitExportError.XML_EXPORT_FAIL)

    # Get current branch if not already set
    if not branch:
        cmd = ['git', 'symbolic-ref', '--short', 'HEAD']
        try:
            branch = cmd_log(cmd, os.path.abspath(rdirp)).strip('\n')
        except subprocess.CalledProcessError as ex:
            log.exception('Failed to get branch from freshly cloned repo: %r',
                          ex.output)
            raise GitExportError(GitExportError.MISSING_BRANCH)

    # Now that we have fresh xml exported, set identity, add
    # everything to git, commit, and push to the right branch.
    ident = {}
    try:
        user = User.objects.get(username=user)
        ident['name'] = user.username
        ident['email'] = user.email
    except User.DoesNotExist:
        # That's ok, just use default ident
        ident = GIT_EXPORT_DEFAULT_IDENT
    time_stamp = timezone.now()
    cwd = os.path.abspath(rdirp)
    commit_msg = 'Export from Studio at {1}'.format(user, time_stamp)
    try:
        cmd_log(['git', 'config', 'user.email', ident['email']], cwd)
        cmd_log(['git', 'config', 'user.name', ident['name']], cwd)
    except subprocess.CalledProcessError as ex:
        log.exception('Error running git configure commands: %r', ex.output)
        raise GitExportError(GitExportError.CONFIG_ERROR)
    try:
        cmd_log(['git', 'add', '.'], cwd)
        cmd_log(['git', 'commit', '-a', '-m', commit_msg], cwd)
    except subprocess.CalledProcessError as ex:
        log.exception('Unable to commit changes: %r', ex.output)
        raise GitExportError(GitExportError.CANNOT_COMMIT)
    try:
        cmd_log(['git', 'push', '-q', 'origin', branch], cwd)
    except subprocess.CalledProcessError as ex:
        log.exception('Error running git push command: %r', ex.output)
        raise GitExportError(GitExportError.CANNOT_PUSH)

########NEW FILE########
__FILENAME__ = check_course
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import check_module_metadata_editability
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    help = '''Enumerates through the course and find common errors'''

    def handle(self, *args, **options):
        if len(args) != 1:
            raise CommandError("check_course requires one argument: <course_id>")

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        store = modulestore()

        course = store.get_course(course_key, depth=3)

        err_cnt = 0

        def _xlint_metadata(module):
            err_cnt = check_module_metadata_editability(module)
            for child in module.get_children():
                err_cnt = err_cnt + _xlint_metadata(child)
            return err_cnt

        err_cnt = err_cnt + _xlint_metadata(course)

        # we've had a bug where the xml_attributes field can we rewritten as a string rather than a dict
        def _check_xml_attributes_field(module):
            err_cnt = 0
            if hasattr(module, 'xml_attributes') and isinstance(module.xml_attributes, basestring):
                print 'module = {0} has xml_attributes as a string. It should be a dict'.format(module.location)
                err_cnt = err_cnt + 1
            for child in module.get_children():
                err_cnt = err_cnt + _check_xml_attributes_field(child)
            return err_cnt

        err_cnt = err_cnt + _check_xml_attributes_field(course)

        # check for dangling discussion items, this can cause errors in the forums
        def _get_discussion_items(module):
            discussion_items = []
            if module.location.category == 'discussion':
                discussion_items = discussion_items + [module.location]

            for child in module.get_children():
                discussion_items = discussion_items + _get_discussion_items(child)

            return discussion_items

        discussion_items = _get_discussion_items(course)

        # now query all discussion items via get_items() and compare with the tree-traversal
        queried_discussion_items = store.get_items(course_key=course_key, category='discussion',)

        for item in queried_discussion_items:
            if item.location not in discussion_items:
                print 'Found dangling discussion module = {0}'.format(item.location)

########NEW FILE########
__FILENAME__ = clone_course
"""
Script for cloning a course
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.store_utilities import clone_course
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from student.roles import CourseInstructorRole, CourseStaffRole
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


#
# To run from command line: ./manage.py cms clone_course --settings=dev master/300/cough edx/111/foo
#
class Command(BaseCommand):
    """Clone a MongoDB-backed course to another location"""
    help = 'Clone a MongoDB backed course to another location'

    def course_key_from_arg(self, arg):
        """
        Convert the command line arg into a course key
        """
        try:
            return CourseKey.from_string(arg)
        except InvalidKeyError:
            return SlashSeparatedCourseKey.from_deprecated_string(arg)

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 2:
            raise CommandError("clone requires 2 arguments: <source-course_id> <dest-course_id>")

        source_course_id = self.course_key_from_arg(args[0])
        dest_course_id = self.course_key_from_arg(args[1])

        mstore = modulestore('direct')
        cstore = contentstore()

        mstore.ignore_write_events_on_courses.add(dest_course_id)

        print("Cloning course {0} to {1}".format(source_course_id, dest_course_id))

        if clone_course(mstore, cstore, source_course_id, dest_course_id):
            print("copying User permissions...")
            # purposely avoids auth.add_user b/c it doesn't have a caller to authorize
            CourseInstructorRole(dest_course_id).add_users(
                *CourseInstructorRole(source_course_id).users_with_role()
            )
            CourseStaffRole(dest_course_id).add_users(
                *CourseStaffRole(source_course_id).users_with_role()
            )

########NEW FILE########
__FILENAME__ = course_id_clash
"""
Script for finding all courses whose org/name pairs == other courses when ignoring case
"""
from django.core.management.base import BaseCommand
from xmodule.modulestore.django import modulestore


#
# To run from command line: ./manage.py cms --settings dev course_id_clash
#
class Command(BaseCommand):
    """
    Script for finding all courses whose org/name pairs == other courses when ignoring case
    """
    help = 'List all courses ids which may collide when ignoring case'

    def handle(self, *args, **options):
        mstore = modulestore()
        if hasattr(mstore, 'collection'):
            map_fn = '''
                function () {
                    emit(this._id.org.toLowerCase()+this._id.course.toLowerCase(), {target: this._id});
                }
            '''
            reduce_fn = '''
                function (idpair, matches) {
                    var result = {target: []};
                    matches.forEach(function (match) {
                        result.target.push(match.target);
                    });
                    return result;
                }
            '''
            finalize = '''
                function(key, reduced) {
                    if (Array.isArray(reduced.target)) {
                        return reduced;
                    }
                    else {return null;}
                    }
            '''
            results = mstore.collection.map_reduce(
                map_fn, reduce_fn, {'inline': True}, query={'_id.category': 'course'}, finalize=finalize
            )
            results = results.get('results')
            for entry in results:
                if entry.get('value') is not None:
                    print '{:-^40}'.format(entry.get('_id'))
                    for course_id in entry.get('value').get('target'):
                        print '   {}/{}/{}'.format(course_id.get('org'), course_id.get('course'), course_id.get('name'))


########NEW FILE########
__FILENAME__ = delete_course
###
### Script for cloning a course
###
from django.core.management.base import BaseCommand, CommandError
from .prompt import query_yes_no
from contentstore.utils import delete_course_and_groups
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    help = '''Delete a MongoDB backed course'''

    def handle(self, *args, **options):
        if len(args) != 1 and len(args) != 2:
            raise CommandError("delete_course requires one or more arguments: <course_id> |commit|")

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        commit = False
        if len(args) == 2:
            commit = args[1] == 'commit'

        if commit:
            print('Actually going to delete the course from DB....')

        if query_yes_no("Deleting course {0}. Confirm?".format(course_key), default="no"):
            if query_yes_no("Are you sure. This action cannot be undone!", default="no"):
                delete_course_and_groups(course_key, commit)

########NEW FILE########
__FILENAME__ = edit_course_tabs
###
### Script for editing the course's tabs
###

#
# Run it this way:
#   ./manage.py cms --settings dev edit_course_tabs --course Stanford/CS99/2013_spring
#
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
from .prompt import query_yes_no

from courseware.courses import get_course_by_id

from contentstore.views import tabs
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.keys import CourseKey


def print_course(course):
    "Prints out the course id and a numbered list of tabs."
    print course.id
    print 'num type name'
    for index, item in enumerate(course.tabs):
        print index + 1, '"' + item.get('type') + '"', '"' + item.get('name', '') + '"'


# course.tabs looks like this
# [{u'type': u'courseware'}, {u'type': u'course_info', u'name': u'Course Info'}, {u'type': u'textbooks'},
# {u'type': u'discussion', u'name': u'Discussion'}, {u'type': u'wiki', u'name': u'Wiki'},
# {u'type': u'progress', u'name': u'Progress'}]


class Command(BaseCommand):
    help = """See and edit a course's tabs list. Only supports insertion
and deletion. Move and rename etc. can be done with a delete
followed by an insert. The tabs are numbered starting with 1.
Tabs 1 and 2 cannot be changed, and tabs of type static_tab cannot
be edited (use Studio for those).

As a first step, run the command with a courseid like this:
  --course Stanford/CS99/2013_spring
This will print the existing tabs types and names. Then run the
command again, adding --insert or --delete to edit the list.
"""
    # Making these option objects separately, so can refer to their .help below
    course_option = make_option('--course',
                                action='store',
                                dest='course',
                                default=False,
                                help='--course <id> required, e.g. Stanford/CS99/2013_spring')
    delete_option = make_option('--delete',
                                action='store_true',
                                dest='delete',
                                default=False,
                                help='--delete <tab-number>')
    insert_option = make_option('--insert',
                                action='store_true',
                                dest='insert',
                                default=False,
                                help='--insert <tab-number> <type> <name>, e.g. 2 "course_info" "Course Info"')

    option_list = BaseCommand.option_list + (course_option, delete_option, insert_option)

    def handle(self, *args, **options):
        if not options['course']:
            raise CommandError(Command.course_option.help)

        try:
            course_key = CourseKey.from_string(options['course'])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(options['course'])

        course = get_course_by_id(course_key)

        print 'Warning: this command directly edits the list of course tabs in mongo.'
        print 'Tabs before any changes:'
        print_course(course)

        try:
            if options['delete']:
                if len(args) != 1:
                    raise CommandError(Command.delete_option.help)
                num = int(args[0])
                if query_yes_no('Deleting tab {0} Confirm?'.format(num), default='no'):
                    tabs.primitive_delete(course, num - 1)  # -1 for 0-based indexing
            elif options['insert']:
                if len(args) != 3:
                    raise CommandError(Command.insert_option.help)
                num = int(args[0])
                tab_type = args[1]
                name = args[2]
                if query_yes_no('Inserting tab {0} "{1}" "{2}" Confirm?'.format(num, tab_type, name), default='no'):
                    tabs.primitive_insert(course, num - 1, tab_type, name)  # -1 as above
        except ValueError as e:
            # Cute: translate to CommandError so the CLI error prints nicely.
            raise CommandError(e)

########NEW FILE########
__FILENAME__ = empty_asset_trashcan
from django.core.management.base import BaseCommand, CommandError
from xmodule.contentstore.utils import empty_asset_trashcan
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from .prompt import query_yes_no
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    help = '''Empty the trashcan. Can pass an optional course_id to limit the damage.'''

    def handle(self, *args, **options):
        if len(args) != 1 and len(args) != 0:
            raise CommandError("empty_asset_trashcan requires one or no arguments: |<course_id>|")

        if len(args) == 1:
            try:
                course_key = CourseKey.from_string(args[0])
            except InvalidKeyError:
                course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

            course_ids = [course_key]
        else:
            course_ids = [course.id for course in modulestore('direct').get_courses()]

        if query_yes_no("Emptying trashcan. Confirm?", default="no"):
            empty_asset_trashcan(course_ids)

########NEW FILE########
__FILENAME__ = export
"""
Script for exporting courseware from Mongo to a tar.gz file
"""
import os

from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.contentstore.django import contentstore
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class Command(BaseCommand):
    """
    Export the specified data directory into the default ModuleStore
    """
    help = 'Export the specified data directory into the default ModuleStore'

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 2:
            raise CommandError("export requires two arguments: <course id> <output path>")

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        output_path = args[1]

        print("Exporting course id = {0} to {1}".format(course_key, output_path))

        root_dir = os.path.dirname(output_path)
        course_dir = os.path.splitext(os.path.basename(output_path))[0]

        export_to_xml(modulestore('direct'), contentstore(), course_key, root_dir, course_dir, modulestore())

########NEW FILE########
__FILENAME__ = export_all_courses
"""
Script for exporting all courseware from Mongo to a directory
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore


class Command(BaseCommand):
    """Export all courses from mongo to the specified data directory"""
    help = 'Export all courses from mongo to the specified data directory'

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 1:
            raise CommandError("export requires one argument: <output path>")

        output_path = args[0]

        cs = contentstore()
        ms = modulestore('direct')
        root_dir = output_path
        courses = ms.get_courses()

        print("%d courses to export:" % len(courses))
        cids = [x.id for x in courses]
        print(cids)

        for course_id in cids:

            print("-"*77)
            print("Exporting course id = {0} to {1}".format(course_id, output_path))

            if 1:
                try:
                    course_dir = course_id.replace('/', '...')
                    export_to_xml(ms, cs, course_id, root_dir, course_dir, modulestore())
                except Exception as err:
                    print("="*30 + "> Oops, failed to export %s" % course_id)
                    print("Error:")
                    print(err)

########NEW FILE########
__FILENAME__ = export_convert_format
"""
Script for converting a tar.gz file representing an exported course
to the archive format used by a different version of export.

Sample invocation: ./manage.py export_convert_format mycourse.tar.gz ~/newformat/
"""
import os
from path import path
from django.core.management.base import BaseCommand, CommandError

from tempfile import mkdtemp
import tarfile
import shutil
from extract_tar import safetar_extractall

from xmodule.modulestore.xml_exporter import convert_between_versions


class Command(BaseCommand):
    """
    Convert between export formats.
    """
    help = 'Convert between versions 0 and 1 of the course export format'
    args = '<tar.gz archive file> <output path>'

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) != 2:
            raise CommandError("export requires two arguments: <tar.gz file> <output path>")

        source_archive = args[0]
        output_path = args[1]

        # Create temp directories to extract the source and create the target archive.
        temp_source_dir = mkdtemp()
        temp_target_dir = mkdtemp()
        try:
            extract_source(source_archive, temp_source_dir)

            desired_version = convert_between_versions(temp_source_dir, temp_target_dir)

            # New zip up the target directory.
            parts = os.path.basename(source_archive).split('.')
            archive_name = path(output_path) / "{source_name}_version_{desired_version}.tar.gz".format(
                source_name=parts[0], desired_version=desired_version
            )
            with open(archive_name, "w"):
                tar_file = tarfile.open(archive_name, mode='w:gz')
                try:
                    for item in os.listdir(temp_target_dir):
                        tar_file.add(path(temp_target_dir) / item, arcname=item)

                finally:
                    tar_file.close()

            print("Created archive {0}".format(archive_name))

        except ValueError as err:
            raise CommandError(err)

        finally:
            shutil.rmtree(temp_source_dir)
            shutil.rmtree(temp_target_dir)


def extract_source(source_archive, target):
    """
    Extract the archive into the given target directory.
    """
    with tarfile.open(source_archive) as tar_file:
        safetar_extractall(tar_file, target)

########NEW FILE########
__FILENAME__ = git_export
"""
This command exports a course from CMS to a git repository.
It takes as arguments the course id to export (i.e MITx/999/2020 ) and
the repository to commit too.  It takes username as an option for identifying
the commit, as well as a directory path to place the git repository.

By default it will use settings.GIT_REPO_EXPORT_DIR/repo_name as the cloned
directory.  It is branch aware, but will reset all local changes to the
repository before attempting to export the XML, add, and commit changes if
any have taken place.

This functionality is also available as an export view in studio if the giturl
attribute is set and the FEATURE['ENABLE_EXPORT_GIT'] is set.
"""

import logging
from optparse import make_option

from django.core.management.base import BaseCommand, CommandError
from django.utils.translation import ugettext as _

import contentstore.git_export_utils as git_export_utils
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from opaque_keys import InvalidKeyError
from contentstore.git_export_utils import GitExportError
from xmodule.modulestore.keys import CourseKey

log = logging.getLogger(__name__)


class Command(BaseCommand):
    """
    Take a course from studio and export it to a git repository.
    """

    option_list = BaseCommand.option_list + (
        make_option('--username', '-u', dest='user',
                    help=('Specify a username from LMS/Studio to be used '
                          'as the commit author.')),
        make_option('--repo_dir', '-r', dest='repo',
                    help='Specify existing git repo directory.'),
    )

    help = _('Take the specified course and attempt to '
             'export it to a git repository\n. Course directory '
             'must already be a git repository. Usage: '
             ' git_export <course_loc> <git_url>')

    def handle(self, *args, **options):
        """
        Checks arguments and runs export function if they are good
        """

        if len(args) != 2:
            raise CommandError('This script requires exactly two arguments: '
                               'course_loc and git_url')

        # Rethrow GitExportError as CommandError for SystemExit
        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            try:
                course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])
            except InvalidKeyError:
                raise CommandError(GitExportError.BAD_COURSE)

        try:
            git_export_utils.export_to_git(
                course_key,
                args[1],
                options.get('user', ''),
                options.get('rdir', None)
            )
        except git_export_utils.GitExportError as ex:
            raise CommandError(str(ex))

########NEW FILE########
__FILENAME__ = import
"""
Script for importing courseware from XML format
"""

from django.core.management.base import BaseCommand, CommandError, make_option
from django_comment_common.utils import (seed_permissions_roles,
                                         are_permissions_roles_seeded)
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore


class Command(BaseCommand):
    """
    Import the specified data directory into the default ModuleStore
    """
    help = 'Import the specified data directory into the default ModuleStore'

    option_list = BaseCommand.option_list + (
        make_option('--nostatic',
                    action='store_true',
                    help='Skip import of static content'),
    )

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) == 0:
            raise CommandError("import requires at least one argument: <data directory> [--nostatic] [<course dir>...]")

        data_dir = args[0]
        do_import_static = not (options.get('nostatic', False))
        if len(args) > 1:
            course_dirs = args[1:]
        else:
            course_dirs = None
        self.stdout.write("Importing.  Data_dir={data}, course_dirs={courses}\n".format(
            data=data_dir,
            courses=course_dirs,
            dis=do_import_static))
        try:
            mstore = modulestore('direct')
        except KeyError:
            self.stdout.write('Unable to load direct modulestore, trying '
                              'default\n')
            mstore = modulestore('default')

        _, course_items = import_from_xml(
            mstore, data_dir, course_dirs, load_error_modules=False,
            static_content_store=contentstore(), verbose=True,
            do_import_static=do_import_static,
            create_new_course=True,
        )

        for course in course_items:
            course_id = course.id
            if not are_permissions_roles_seeded(course_id):
                self.stdout.write('Seeding forum roles for course {0}\n'.format(course_id))
                seed_permissions_roles(course_id)

########NEW FILE########
__FILENAME__ = migrate_to_split
"""
Django management command to migrate a course from the old Mongo modulestore
to the new split-Mongo modulestore.
"""
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.split_migrator import SplitMigrator
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.keys import CourseKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import SlashSeparatedCourseKey


def user_from_str(identifier):
    """
    Return a user identified by the given string. The string could be an email
    address, or a stringified integer corresponding to the ID of the user in
    the database. If no user could be found, a User.DoesNotExist exception
    will be raised.
    """
    try:
        user_id = int(identifier)
    except ValueError:
        return User.objects.get(email=identifier)
    else:
        return User.objects.get(id=user_id)


class Command(BaseCommand):
    "Migrate a course from old-Mongo to split-Mongo"

    help = "Migrate a course from old-Mongo to split-Mongo"
    args = "course_key email <new org> <new offering>"

    def parse_args(self, *args):
        """
        Return a 4-tuple of (course_key, user, org, offering).
        If the user didn't specify an org & offering, those will be None.
        """
        if len(args) < 2:
            raise CommandError(
                "migrate_to_split requires at least two arguments: "
                "a course_key and a user identifier (email or ID)"
            )

        try:
            course_key = CourseKey.from_string(args[0])
        except InvalidKeyError:
            course_key = SlashSeparatedCourseKey.from_deprecated_string(args[0])

        try:
            user = user_from_str(args[1])
        except User.DoesNotExist:
            raise CommandError("No user found identified by {}".format(args[1]))

        try:
            org = args[2]
            offering = args[3]
        except IndexError:
            org = offering = None

        return course_key, user, org, offering

    def handle(self, *args, **options):
        course_key, user, org, offering = self.parse_args(*args)

        migrator = SplitMigrator(
            draft_modulestore=modulestore('default'),
            direct_modulestore=modulestore('direct'),
            split_modulestore=modulestore('split'),
            loc_mapper=loc_mapper(),
        )

        migrator.migrate_mongo_course(course_key, user, org, offering)

########NEW FILE########
__FILENAME__ = populate_creators
"""
Script for granting existing course instructors course creator privileges.

This script is only intended to be run once on a given environment.
"""
from course_creators.views import add_user_with_status_granted, add_user_with_status_unrequested
from django.core.management.base import BaseCommand

from django.contrib.auth.models import User
from django.db.utils import IntegrityError
from student.roles import CourseInstructorRole, CourseStaffRole

#------------ to run: ./manage.py cms populate_creators --settings=dev
class Command(BaseCommand):
    """
    Script for granting existing course instructors course creator privileges.
    """
    help = 'Grants all users with INSTRUCTOR role permission to create courses'

    def handle(self, *args, **options):
        """
        The logic of the command.
        """
        username = 'populate_creators_command'
        email = 'grant+creator+access@edx.org'
        try:
            admin = User.objects.create_user(username, email, 'foo')
            admin.is_staff = True
            admin.save()
        except IntegrityError:
            # If the script did not complete the last time it was run,
            # the admin user will already exist.
            admin = User.objects.get(username=username, email=email)

        for user in get_users_with_role(CourseInstructorRole.ROLE):
            add_user_with_status_granted(admin, user)

        # Some users will be both staff and instructors. Those folks have been
        # added with status granted above, and add_user_with_status_unrequested
        # will not try to add them again if they already exist in the course creator database.
        for user in get_users_with_role(CourseStaffRole.ROLE):
            add_user_with_status_unrequested(user)

        # There could be users who are not in either staff or instructor (they've
        # never actually done anything in Studio). I plan to add those as unrequested
        # when they first go to their dashboard.

        admin.delete()


#=============================================================================================================
# Because these are expensive and far-reaching, I moved them here
def get_users_with_role(role_prefix):
    """
    An expensive operation which finds all users in the db with the given role prefix
    """
    return User.objects.filter(groups__name__startswith=role_prefix)

########NEW FILE########
__FILENAME__ = prompt
import sys


def query_yes_no(question, default="yes"):
    """Ask a yes/no question via raw_input() and return their answer.

    "question" is a string that is presented to the user.
    "default" is the presumed answer if the user just hits <Enter>.
        It must be "yes" (the default), "no" or None (meaning
        an answer is required of the user).

    The "answer" return value is one of "yes" or "no".
    """
    valid = {"yes": True,   "y": True,  "ye": True,
             "no": False,     "n": False}
    if default is None:
        prompt = " [y/n] "
    elif default == "yes":
        prompt = " [Y/n] "
    elif default == "no":
        prompt = " [y/N] "
    else:
        raise ValueError("invalid default answer: '%s'" % default)

    while True:
        sys.stdout.write(question + prompt)
        choice = raw_input().lower()
        if default is not None and choice == '':
            return valid[default]
        elif choice in valid:
            return valid[choice]
        else:
            sys.stdout.write("Please respond with 'yes' or 'no' (or 'y' or 'n').\n")

########NEW FILE########
__FILENAME__ = restore_asset_from_trashcan
from django.core.management.base import BaseCommand, CommandError
from xmodule.contentstore.utils import restore_asset_from_trashcan


class Command(BaseCommand):
    help = '''Restore a deleted asset from the trashcan back to it's original course'''

    def handle(self, *args, **options):
        if len(args) != 1 and len(args) != 0:
            raise CommandError("restore_asset_from_trashcan requires one argument: <location>")

        restore_asset_from_trashcan(args[0])


########NEW FILE########
__FILENAME__ = rollback_split_course
"""
Django management command to rollback a migration to split. The way to do this
is to delete the course from the split mongo datastore.
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.locator import CourseLocator


class Command(BaseCommand):
    "Rollback a course that was migrated to the split Mongo datastore"

    help = "Rollback a course that was migrated to the split Mongo datastore"
    args = "org offering"

    def handle(self, *args, **options):
        if len(args) < 2:
            raise CommandError(
                "rollback_split_course requires 2 arguments (org offering)"
            )

        try:
            locator = CourseLocator(org=args[0], offering=args[1])
        except ValueError:
            raise CommandError("Invalid org or offering string {}, {}".format(*args))

        location = loc_mapper().translate_locator_to_location(locator, get_course=True)
        if not location:
            raise CommandError(
                "This course does not exist in the old Mongo store. "
                "This command is designed to rollback a course, not delete "
                "it entirely."
            )
        old_mongo_course = modulestore('direct').get_item(location)
        if not old_mongo_course:
            raise CommandError(
                "This course does not exist in the old Mongo store. "
                "This command is designed to rollback a course, not delete "
                "it entirely."
            )

        try:
            modulestore('split').delete_course(locator)
        except ItemNotFoundError:
            raise CommandError("No course found with locator {}".format(locator))

        print(
            'Course rolled back successfully. To delete this course entirely, '
            'call the "delete_course" management command.'
        )

########NEW FILE########
__FILENAME__ = test_course_id_clash
import sys
from StringIO import StringIO
from django.test import TestCase
from django.core.management import call_command
from xmodule.modulestore.tests.factories import CourseFactory

class ClashIdTestCase(TestCase):
    """
    Test for course_id_clash.
    """
    def test_course_clash(self):
        """
        Test for course_id_clash.
        """
        expected = []
        # clashing courses
        course = CourseFactory.create(org="test", course="courseid", display_name="run1")
        expected.append(course.id)
        course = CourseFactory.create(org="TEST", course="courseid", display_name="RUN12")
        expected.append(course.id)
        course = CourseFactory.create(org="test", course="CourseId", display_name="aRUN123")
        expected.append(course.id)
        # not clashing courses
        not_expected = []
        course = CourseFactory.create(org="test", course="course2", display_name="run1")
        not_expected.append(course.id)
        course = CourseFactory.create(org="test1", course="courseid", display_name="run1")
        not_expected.append(course.id)
        course = CourseFactory.create(org="test", course="courseid0", display_name="run1")
        not_expected.append(course.id)

        old_stdout = sys.stdout
        sys.stdout = mystdout = StringIO()
        call_command('course_id_clash', stdout=mystdout)
        sys.stdout = old_stdout
        result = mystdout.getvalue()
        for courseid in expected:
            self.assertIn(courseid.to_deprecated_string(), result)
        for courseid in not_expected:
            self.assertNotIn(courseid.to_deprecated_string(), result)

########NEW FILE########
__FILENAME__ = test_export_convert_format
"""
Test for export_convert_format.
"""
from unittest import TestCase
from django.core.management import call_command, CommandError
from tempfile import mkdtemp
import shutil
from path import path
from contentstore.management.commands.export_convert_format import Command, extract_source
from xmodule.tests.helpers import directories_equal


class ConvertExportFormat(TestCase):
    """
    Tests converting between export formats.
    """
    def setUp(self):
        """ Common setup. """
        self.temp_dir = mkdtemp()
        self.data_dir = path(__file__).realpath().parent / 'data'
        self.version0 = self.data_dir / "Version0_drafts.tar.gz"
        self.version1 = self.data_dir / "Version1_drafts.tar.gz"

        self.command = Command()

    def tearDown(self):
        """ Common cleanup. """
        shutil.rmtree(self.temp_dir)

    def test_no_args(self):
        """ Test error condition of no arguments. """
        errstring = "export requires two arguments"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle()

    def test_version1_archive(self):
        """
        Smoke test for creating a version 1 archive from a version 0.
        """
        call_command('export_convert_format', self.version0, self.temp_dir)
        output = path(self.temp_dir) / 'Version0_drafts_version_1.tar.gz'
        self.assertTrue(self._verify_archive_equality(output, self.version1))

    def test_version0_archive(self):
        """
        Smoke test for creating a version 0 archive from a version 1.
        """
        call_command('export_convert_format', self.version1, self.temp_dir)
        output = path(self.temp_dir) / 'Version1_drafts_version_0.tar.gz'
        self.assertTrue(self._verify_archive_equality(output, self.version0))

    def _verify_archive_equality(self, file1, file2):
        """
        Helper function for determining if 2 archives are equal.
        """
        temp_dir_1 = mkdtemp()
        temp_dir_2 = mkdtemp()
        try:
            extract_source(file1, temp_dir_1)
            extract_source(file2, temp_dir_2)
            return directories_equal(temp_dir_1, temp_dir_2)

        finally:
            shutil.rmtree(temp_dir_1)
            shutil.rmtree(temp_dir_2)

########NEW FILE########
__FILENAME__ = test_git_export
"""
Unittests for exporting to git via management command.
"""

import copy
import os
import shutil
import StringIO
import subprocess
import unittest
from uuid import uuid4

from django.conf import settings
from django.core.management import call_command
from django.core.management.base import CommandError
from django.test.utils import override_settings

from contentstore.tests.utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils
from contentstore.git_export_utils import GitExportError
from xmodule.modulestore.locations import SlashSeparatedCourseKey

FEATURES_WITH_EXPORT_GIT = settings.FEATURES.copy()
FEATURES_WITH_EXPORT_GIT['ENABLE_EXPORT_GIT'] = True
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
@override_settings(FEATURES=FEATURES_WITH_EXPORT_GIT)
class TestGitExport(CourseTestCase):
    """
    Excercise the git_export django management command with various inputs.
    """

    def setUp(self):
        """
        Create/reinitialize bare repo and folders needed
        """
        super(TestGitExport, self).setUp()

        if not os.path.isdir(git_export_utils.GIT_REPO_EXPORT_DIR):
            os.mkdir(git_export_utils.GIT_REPO_EXPORT_DIR)
            self.addCleanup(shutil.rmtree, git_export_utils.GIT_REPO_EXPORT_DIR)

        self.bare_repo_dir = '{0}/data/test_bare.git'.format(
            os.path.abspath(settings.TEST_ROOT))
        if not os.path.isdir(self.bare_repo_dir):
            os.mkdir(self.bare_repo_dir)
            self.addCleanup(shutil.rmtree, self.bare_repo_dir)
        subprocess.check_output(['git', '--bare', 'init'],
                                cwd=self.bare_repo_dir)

    def test_command(self):
        """
        Test that the command interface works. Ignore stderr for clean
        test output.
        """
        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, 'This script requires.*'):
                call_command('git_export', 'blah', 'blah', 'blah',
                             stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)

        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, 'This script requires.*'):
                call_command('git_export', stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)

        # Send bad url to get course not exported
        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, GitExportError.URL_BAD):
                call_command('git_export', 'foo/bar/baz', 'silly',
                             stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)
        # Send bad course_id to get course not exported
        with self.assertRaises(SystemExit) as ex:
            with self.assertRaisesRegexp(CommandError, GitExportError.BAD_COURSE):
                call_command('git_export', 'foo/bar:baz', 'silly',
                             stderr=StringIO.StringIO())
        self.assertEqual(ex.exception.code, 1)

    def test_bad_git_url(self):
        """
        Test several bad URLs for validation
        """
        course_key = SlashSeparatedCourseKey('org', 'course', 'run')
        with self.assertRaisesRegexp(GitExportError, str(GitExportError.URL_BAD)):
            git_export_utils.export_to_git(course_key, 'Sillyness')

        with self.assertRaisesRegexp(GitExportError, str(GitExportError.URL_BAD)):
            git_export_utils.export_to_git(course_key, 'example.com:edx/notreal')

        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.URL_NO_AUTH)):
            git_export_utils.export_to_git(course_key, 'http://blah')

    def test_bad_git_repos(self):
        """
        Test invalid git repos
        """
        test_repo_path = '{}/test_repo'.format(git_export_utils.GIT_REPO_EXPORT_DIR)
        self.assertFalse(os.path.isdir(test_repo_path))
        course_key = SlashSeparatedCourseKey('foo', 'blah', '100-')
        # Test bad clones
        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.CANNOT_PULL)):
            git_export_utils.export_to_git(
                course_key,
                'https://user:blah@example.com/test_repo.git')
        self.assertFalse(os.path.isdir(test_repo_path))

        # Setup good repo with bad course to test xml export
        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.XML_EXPORT_FAIL)):
            git_export_utils.export_to_git(
                course_key,
                'file://{0}'.format(self.bare_repo_dir))

        # Test bad git remote after successful clone
        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.CANNOT_PULL)):
            git_export_utils.export_to_git(
                course_key,
                'https://user:blah@example.com/r.git')

    @unittest.skipIf(os.environ.get('GIT_CONFIG') or
                     os.environ.get('GIT_AUTHOR_EMAIL') or
                     os.environ.get('GIT_AUTHOR_NAME') or
                     os.environ.get('GIT_COMMITTER_EMAIL') or
                     os.environ.get('GIT_COMMITTER_NAME'),
                     'Global git override set')
    def test_git_ident(self):
        """
        Test valid course with and without user specified.

        Test skipped if git global config override environment variable GIT_CONFIG
        is set.
        """
        git_export_utils.export_to_git(
            self.course.id,
            'file://{0}'.format(self.bare_repo_dir),
            'enigma'
        )
        expect_string = '{0}|{1}\n'.format(
            git_export_utils.GIT_EXPORT_DEFAULT_IDENT['name'],
            git_export_utils.GIT_EXPORT_DEFAULT_IDENT['email']
        )
        cwd = os.path.abspath(git_export_utils.GIT_REPO_EXPORT_DIR / 'test_bare')
        git_log = subprocess.check_output(['git', 'log', '-1',
                                           '--format=%an|%ae'], cwd=cwd)
        self.assertEqual(expect_string, git_log)

        # Make changes to course so there is something to commit
        self.populate_course()
        git_export_utils.export_to_git(
            self.course.id,
            'file://{0}'.format(self.bare_repo_dir),
            self.user.username
        )
        expect_string = '{0}|{1}\n'.format(
            self.user.username,
            self.user.email,
        )
        git_log = subprocess.check_output(
            ['git', 'log', '-1', '--format=%an|%ae'], cwd=cwd)
        self.assertEqual(expect_string, git_log)

    def test_no_change(self):
        """
        Test response if there are no changes
        """
        git_export_utils.export_to_git(
            self.course.id,
            'file://{0}'.format(self.bare_repo_dir)
        )

        with self.assertRaisesRegexp(GitExportError,
                                     str(GitExportError.CANNOT_COMMIT)):
            git_export_utils.export_to_git(
                self.course.id, 'file://{0}'.format(self.bare_repo_dir))

########NEW FILE########
__FILENAME__ = test_import
"""
Unittests for importing a course via management command
"""

import os
from path import path
import shutil
import tempfile

from django.core.management import call_command
from django.test.utils import override_settings

from contentstore.tests.modulestore_config import TEST_MODULESTORE
from django_comment_common.utils import are_permissions_roles_seeded
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.locations import SlashSeparatedCourseKey


@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestImport(ModuleStoreTestCase):
    """
    Unit tests for importing a course from command line
    """

    BASE_COURSE_KEY = SlashSeparatedCourseKey(u'edX', u'test_import_course', u'2013_Spring')
    DIFF_KEY = SlashSeparatedCourseKey(u'edX', u'test_import_course', u'2014_Spring')
    TRUNCATED_KEY = SlashSeparatedCourseKey(u'edX', u'test_import', u'2014_Spring')

    def create_course_xml(self, content_dir, course_id):
        directory = tempfile.mkdtemp(dir=content_dir)
        os.makedirs(os.path.join(directory, "course"))
        with open(os.path.join(directory, "course.xml"), "w+") as f:
            f.write('<course url_name="{0.run}" org="{0.org}" '
                    'course="{0.course}"/>'.format(course_id))

        with open(os.path.join(directory, "course", "{0.run}.xml".format(course_id)), "w+") as f:
            f.write('<course></course>')

        return directory

    def setUp(self):
        """
        Build course XML for importing
        """
        super(TestImport, self).setUp()
        self.content_dir = path(tempfile.mkdtemp())
        self.addCleanup(shutil.rmtree, self.content_dir)

        # Create good course xml
        self.good_dir = self.create_course_xml(self.content_dir, self.BASE_COURSE_KEY)

        # Create run changed course xml
        self.dupe_dir = self.create_course_xml(self.content_dir, self.DIFF_KEY)

        # Create course XML where TRUNCATED_COURSE.org == BASE_COURSE_ID.org
        # and BASE_COURSE_ID.startswith(TRUNCATED_COURSE.course)
        self.course_dir = self.create_course_xml(self.content_dir, self.TRUNCATED_KEY)

    def test_forum_seed(self):
        """
        Tests that forum roles were created with import.
        """
        self.assertFalse(are_permissions_roles_seeded(self.BASE_COURSE_KEY))
        call_command('import', self.content_dir, self.good_dir)
        self.assertTrue(are_permissions_roles_seeded(self.BASE_COURSE_KEY))

    def test_duplicate_with_url(self):
        """
        Check to make sure an import doesn't import courses that have the
        same org and course, but they have different runs in order to
        prevent modulestore "findone" exceptions on deletion
        """
        # Load up base course and verify it is available
        call_command('import', self.content_dir, self.good_dir)
        store = modulestore()
        self.assertIsNotNone(store.get_course(self.BASE_COURSE_KEY))

        # Now load up duped course and verify it doesn't load
        call_command('import', self.content_dir, self.dupe_dir)
        self.assertIsNone(store.get_course(self.DIFF_KEY))

    def test_truncated_course_with_url(self):
        """
        Check to make sure an import only blocks true duplicates: new
        courses with similar but not unique org/course combinations aren't
        blocked if the original course's course starts with the new course's
        org/course combinations (i.e. EDx/0.00x/Spring -> EDx/0.00/Spring)
        """
        # Load up base course and verify it is available
        call_command('import', self.content_dir, self.good_dir)
        store = modulestore()
        self.assertIsNotNone(store.get_course(self.BASE_COURSE_KEY))

        # Now load up the course with a similar course_id and verify it loads
        call_command('import', self.content_dir, self.course_dir)
        self.assertIsNotNone(store.get_course(self.TRUNCATED_KEY))

########NEW FILE########
__FILENAME__ = test_migrate_to_split
"""
Unittests for migrating a course to split mongo
"""
import unittest

from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.migrate_to_split import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, clear_existing_modulestores
from xmodule.modulestore.locator import CourseLocator
# pylint: disable=E1101


@unittest.skip("Not fixing split mongo until we land this long branch")
class TestArgParsing(unittest.TestCase):
    """
    Tests for parsing arguments for the `migrate_to_split` management command
    """
    def setUp(self):
        self.command = Command()

    def test_no_args(self):
        errstring = "migrate_to_split requires at least two arguments"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle()

    def test_invalid_location(self):
        errstring = "Invalid location string"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("foo", "bar")

    def test_nonexistant_user_id(self):
        errstring = "No user found identified by 99"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("i4x://org/course/category/name", "99")

    def test_nonexistant_user_email(self):
        errstring = "No user found identified by fake@example.com"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("i4x://org/course/category/name", "fake@example.com")


@unittest.skip("Not fixing split mongo until we land this long branch")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestMigrateToSplit(ModuleStoreTestCase):
    """
    Unit tests for migrating a course from old mongo to split mongo
    """

    def setUp(self):
        super(TestMigrateToSplit, self).setUp()
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'
        self.user = User.objects.create_user(uname, email, password)
        self.course = CourseFactory()
        self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
        self.addCleanup(clear_existing_modulestores)

    def test_user_email(self):
        call_command(
            "migrate_to_split",
            str(self.course.location),
            str(self.user.email),
        )
        course_from_split = modulestore('split').get_course(self.course.id)
        self.assertIsNotNone(course_from_split)

    def test_user_id(self):
        call_command(
            "migrate_to_split",
            str(self.course.location),
            str(self.user.id),
        )
        course_from_split = modulestore('split').get_course(self.course.id)
        self.assertIsNotNone(course_from_split)

    def test_locator_string(self):
        call_command(
            "migrate_to_split",
            str(self.course.location),
            str(self.user.id),
            "org.dept+name.run",
        )
        locator = CourseLocator(org="org.dept", offering="name.run", branch="published")
        course_from_split = modulestore('split').get_course(locator)
        self.assertIsNotNone(course_from_split)

########NEW FILE########
__FILENAME__ = test_rollback_split_course
"""
Unittests for deleting a split mongo course
"""
import unittest
from StringIO import StringIO
from mock import patch

from django.contrib.auth.models import User
from django.core.management import CommandError, call_command
from django.test.utils import override_settings
from contentstore.management.commands.rollback_split_course import Command
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.persistent_factories import PersistentCourseFactory
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.django import modulestore, loc_mapper
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.split_migrator import SplitMigrator
# pylint: disable=E1101


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
class TestArgParsing(unittest.TestCase):
    """
    Tests for parsing arguments for the `rollback_split_course` management command
    """
    def setUp(self):
        self.command = Command()

    def test_no_args(self):
        errstring = "rollback_split_course requires at least one argument"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle()

    def test_invalid_locator(self):
        errstring = "Invalid locator string !?!"
        with self.assertRaisesRegexp(CommandError, errstring):
            self.command.handle("!?!")


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoOldMongo(ModuleStoreTestCase):
    """
    Unit tests for rolling back a split-mongo course from command line,
    where the course doesn't exist in the old mongo store
    """

    def setUp(self):
        super(TestRollbackSplitCourseNoOldMongo, self).setUp()
        self.course = PersistentCourseFactory()

    def test_no_old_course(self):
        locator = self.course.location
        errstring = "course does not exist in the old Mongo store"
        with self.assertRaisesRegexp(CommandError, errstring):
            Command().handle(str(locator))


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourseNoSplitMongo(ModuleStoreTestCase):
    """
    Unit tests for rolling back a split-mongo course from command line,
    where the course doesn't exist in the split mongo store
    """

    def setUp(self):
        super(TestRollbackSplitCourseNoSplitMongo, self).setUp()
        self.old_course = CourseFactory()

    def test_nonexistent_locator(self):
        locator = loc_mapper().translate_location(self.old_course.location)
        errstring = "No course found with locator"
        with self.assertRaisesRegexp(CommandError, errstring):
            Command().handle(str(locator))


@unittest.skip("Not fixing split mongo until we land opaque-keys 0.9")
@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestRollbackSplitCourse(ModuleStoreTestCase):
    """
    Unit tests for rolling back a split-mongo course from command line
    """
    def setUp(self):
        super(TestRollbackSplitCourse, self).setUp()
        self.old_course = CourseFactory()
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'
        self.user = User.objects.create_user(uname, email, password)

        # migrate old course to split
        migrator = SplitMigrator(
            draft_modulestore=modulestore('default'),
            direct_modulestore=modulestore('direct'),
            split_modulestore=modulestore('split'),
            loc_mapper=loc_mapper(),
        )
        migrator.migrate_mongo_course(self.old_course.location, self.user)
        self.course = modulestore('split').get_course(self.old_course.id)

    @patch("sys.stdout", new_callable=StringIO)
    def test_happy_path(self, mock_stdout):
        course_id = self.course.id
        call_command(
            "rollback_split_course",
            str(course_id),
        )
        with self.assertRaises(ItemNotFoundError):
            modulestore('split').get_course(course_id)

        self.assertIn("Course rolled back successfully", mock_stdout.getvalue())


########NEW FILE########
__FILENAME__ = xlint
"""
Verify the structure of courseware as to it's suitability for import
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_importer import perform_xlint


class Command(BaseCommand):
    """Verify the structure of courseware as to it's suitability for import"""
    help = "Verify the structure of courseware as to it's suitability for import"

    def handle(self, *args, **options):
        "Execute the command"
        if len(args) == 0:
            raise CommandError("import requires at least one argument: <data directory> [<course dir>...]")

        data_dir = args[0]
        if len(args) > 1:
            course_dirs = args[1:]
        else:
            course_dirs = None
        print("Importing.  Data_dir={data}, course_dirs={courses}".format(
            data=data_dir,
            courses=course_dirs))
        perform_xlint(data_dir, course_dirs, load_error_modules=False)

########NEW FILE########
__FILENAME__ = modulestore_config
"""
Define test configuration for modulestores.
"""

from xmodule.modulestore.tests.django_utils import studio_store_config
from django.conf import settings

TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")

########NEW FILE########
__FILENAME__ = tests
"""
This test file will test registration, login, activation, and session activity timeouts
"""
import time
import mock
import unittest

from django.test.utils import override_settings
from django.core.cache import cache
from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse

from contentstore.tests.utils import parse_json, user, registration, AjaxEnabledTestClient
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.test_course_settings import CourseTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
import datetime
from pytz import UTC

from freezegun import freeze_time


@override_settings(MODULESTORE=TEST_MODULESTORE)
class ContentStoreTestCase(ModuleStoreTestCase):
    def _login(self, email, password):
        """
        Login.  View should always return 200.  The success/fail is in the
        returned json
        """
        resp = self.client.post(
            reverse('login_post'),
            {'email': email, 'password': password}
        )
        self.assertEqual(resp.status_code, 200)
        return resp

    def login(self, email, password):
        """Login, check that it worked."""
        resp = self._login(email, password)
        data = parse_json(resp)
        self.assertTrue(data['success'])
        return resp

    def _create_account(self, username, email, password):
        """Try to create an account.  No error checking"""
        resp = self.client.post('/create_account', {
            'username': username,
            'email': email,
            'password': password,
            'location': 'home',
            'language': 'Franglish',
            'name': 'Fred Weasley',
            'terms_of_service': 'true',
            'honor_code': 'true',
        })
        return resp

    def create_account(self, username, email, password):
        """Create the account and check that it worked"""
        resp = self._create_account(username, email, password)
        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertEqual(data['success'], True)

        # Check both that the user is created, and inactive
        self.assertFalse(user(email).is_active)

        return resp

    def _activate_user(self, email):
        """Look up the activation key for the user, then hit the activate view.
        No error checking"""
        activation_key = registration(email).activation_key

        # and now we try to activate
        resp = self.client.get(reverse('activate', kwargs={'key': activation_key}))
        return resp

    def activate_user(self, email):
        resp = self._activate_user(email)
        self.assertEqual(resp.status_code, 200)
        # Now make sure that the user is now actually activated
        self.assertTrue(user(email).is_active)


class AuthTestCase(ContentStoreTestCase):
    """Check that various permissions-related things work"""

    def setUp(self):
        self.email = 'a@b.com'
        self.pw = 'xyz'
        self.username = 'testuser'
        self.client = AjaxEnabledTestClient()
        # clear the cache so ratelimiting won't affect these tests
        cache.clear()

    def check_page_get(self, url, expected):
        resp = self.client.get_html(url)
        self.assertEqual(resp.status_code, expected)
        return resp

    def test_public_pages_load(self):
        """Make sure pages that don't require login load without error."""
        pages = (
            reverse('login'),
            reverse('signup'),
        )
        for page in pages:
            print("Checking '{0}'".format(page))
            self.check_page_get(page, 200)

    def test_create_account_errors(self):
        # No post data -- should fail
        resp = self.client.post('/create_account', {})
        self.assertEqual(resp.status_code, 400)
        data = parse_json(resp)
        self.assertEqual(data['success'], False)

    def test_create_account(self):
        self.create_account(self.username, self.email, self.pw)
        self.activate_user(self.email)

    def test_create_account_username_already_exists(self):
        User.objects.create_user(self.username, self.email, self.pw)
        resp = self._create_account(self.username, "abc@def.com", "password")
        # we have a constraint on unique usernames, so this should fail
        self.assertEqual(resp.status_code, 400)

    def test_create_account_pw_already_exists(self):
        User.objects.create_user(self.username, self.email, self.pw)
        resp = self._create_account("abcdef", "abc@def.com", self.pw)
        # we can have two users with the same password, so this should succeed
        self.assertEqual(resp.status_code, 200)

    @unittest.skipUnless(settings.SOUTH_TESTS_MIGRATE, "South migrations required")
    def test_create_account_email_already_exists(self):
        User.objects.create_user(self.username, self.email, self.pw)
        resp = self._create_account("abcdef", self.email, "password")
        # This is tricky. Django's user model doesn't have a constraint on
        # unique email addresses, but we *add* that constraint during the
        # migration process:
        # see common/djangoapps/student/migrations/0004_add_email_index.py
        #
        # The behavior we *want* is for this account creation request
        # to fail, due to this uniqueness constraint, but the request will
        # succeed if the migrations have not run.
        self.assertEqual(resp.status_code, 400)

    def test_login(self):
        self.create_account(self.username, self.email, self.pw)

        # Not activated yet.  Login should fail.
        resp = self._login(self.email, self.pw)
        data = parse_json(resp)
        self.assertFalse(data['success'])

        self.activate_user(self.email)

        # Now login should work
        self.login(self.email, self.pw)

    def test_login_ratelimited(self):
        # try logging in 30 times, the default limit in the number of failed
        # login attempts in one 5 minute period before the rate gets limited
        for i in xrange(30):
            resp = self._login(self.email, 'wrong_password{0}'.format(i))
            self.assertEqual(resp.status_code, 200)
        resp = self._login(self.email, 'wrong_password')
        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertFalse(data['success'])
        self.assertIn('Too many failed login attempts.', data['value'])

    @override_settings(MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED=3)
    @override_settings(MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS=2)
    def test_excessive_login_failures(self):
        # try logging in 3 times, the account should get locked for 3 seconds
        # note we want to keep the lockout time short, so we don't slow down the tests

        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MAX_FAILED_LOGIN_ATTEMPTS': True}):
            self.create_account(self.username, self.email, self.pw)
            self.activate_user(self.email)

            for i in xrange(3):
                resp = self._login(self.email, 'wrong_password{0}'.format(i))
                self.assertEqual(resp.status_code, 200)
                data = parse_json(resp)
                self.assertFalse(data['success'])
                self.assertIn(
                    'Email or password is incorrect.',
                    data['value']
                )

            # now the account should be locked

            resp = self._login(self.email, 'wrong_password')
            self.assertEqual(resp.status_code, 200)
            data = parse_json(resp)
            self.assertFalse(data['success'])
            self.assertIn(
                'This account has been temporarily locked due to excessive login failures. Try again later.',
                data['value']
            )

            with freeze_time('2100-01-01'):
                self.login(self.email, self.pw)

            # make sure the failed attempt counter gets reset on successful login
            resp = self._login(self.email, 'wrong_password')
            self.assertEqual(resp.status_code, 200)
            data = parse_json(resp)
            self.assertFalse(data['success'])

            # account should not be locked out after just one attempt
            self.login(self.email, self.pw)

            # do one more login when there is no bad login counter row at all in the database to
            # test the "ObjectNotFound" case
            self.login(self.email, self.pw)

    def test_login_link_on_activation_age(self):
        self.create_account(self.username, self.email, self.pw)
        # we want to test the rendering of the activation page when the user isn't logged in
        self.client.logout()
        resp = self._activate_user(self.email)
        self.assertEqual(resp.status_code, 200)

        # check the the HTML has links to the right login page. Note that this is merely a content
        # check and thus could be fragile should the wording change on this page
        expected = 'You can now <a href="' + reverse('login') + '">login</a>.'
        self.assertIn(expected, resp.content)

    def test_private_pages_auth(self):
        """Make sure pages that do require login work."""
        auth_pages = (
            '/course/',
        )

        # These are pages that should just load when the user is logged in
        # (no data needed)
        simple_auth_pages = (
            '/course/',
        )

        # need an activated user
        self.test_create_account()

        # Create a new session
        self.client = AjaxEnabledTestClient()

        # Not logged in.  Should redirect to login.
        print('Not logged in')
        for page in auth_pages:
            print("Checking '{0}'".format(page))
            self.check_page_get(page, expected=302)

        # Logged in should work.
        self.login(self.email, self.pw)

        print('Logged in')
        for page in simple_auth_pages:
            print("Checking '{0}'".format(page))
            self.check_page_get(page, expected=200)

    def test_index_auth(self):

        # not logged in.  Should return a redirect.
        resp = self.client.get_html('/course/')
        self.assertEqual(resp.status_code, 302)

        # Logged in should work.

    @override_settings(SESSION_INACTIVITY_TIMEOUT_IN_SECONDS=1)
    def test_inactive_session_timeout(self):
        """
        Verify that an inactive session times out and redirects to the
        login page
        """
        self.create_account(self.username, self.email, self.pw)
        self.activate_user(self.email)

        self.login(self.email, self.pw)

        # make sure we can access courseware immediately
        course_url = '/course/'
        resp = self.client.get_html(course_url)
        self.assertEquals(resp.status_code, 200)

        # then wait a bit and see if we get timed out
        time.sleep(2)

        resp = self.client.get_html(course_url)

        # re-request, and we should get a redirect to login page
        self.assertRedirects(resp, settings.LOGIN_REDIRECT_URL + '?next=/course/')


class ForumTestCase(CourseTestCase):
    def setUp(self):
        """ Creates the test course. """
        super(ForumTestCase, self).setUp()
        self.course = CourseFactory.create(org='testX', number='727', display_name='Forum Course')

    def test_blackouts(self):
        now = datetime.datetime.now(UTC)
        times1 = [
            (now - datetime.timedelta(days=14), now - datetime.timedelta(days=11)),
            (now + datetime.timedelta(days=24), now + datetime.timedelta(days=30))
        ]
        self.course.discussion_blackouts = [(t.isoformat(), t2.isoformat()) for t, t2 in times1]
        self.assertTrue(self.course.forum_posts_allowed)
        times2 = [
            (now - datetime.timedelta(days=14), now + datetime.timedelta(days=2)),
            (now + datetime.timedelta(days=24), now + datetime.timedelta(days=30))
        ]
        self.course.discussion_blackouts = [(t.isoformat(), t2.isoformat()) for t, t2 in times2]
        self.assertFalse(self.course.forum_posts_allowed)

########NEW FILE########
__FILENAME__ = test_contentstore
# -*- coding: utf-8 -*-
# pylint: disable=E1101

import copy
import mock
import re
import shutil

from datetime import timedelta
from fs.osfs import OSFS
from json import loads
from path import path
from pymongo import MongoClient
from tempdir import mkdtemp_clean
from textwrap import dedent
from uuid import uuid4

from django.conf import settings
from django.contrib.auth.models import User
from django.test import TestCase
from django.test.utils import override_settings

from contentstore.utils import get_modulestore
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import parse_json, AjaxEnabledTestClient
from contentstore.views.component import ADVANCED_COMPONENT_TYPES

from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan
from xmodule.exceptions import NotFoundError, InvalidVersionError
from xmodule.modulestore import mongo
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.inheritance import own_metadata
from xmodule.modulestore.keys import UsageKey
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.store_utilities import clone_course, delete_course
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint

from xmodule.capa_module import CapaDescriptor
from xmodule.course_module import CourseDescriptor
from xmodule.seq_module import SequenceDescriptor

from contentstore.utils import delete_course_and_groups, reverse_url, reverse_course_url
from django_comment_common.utils import are_permissions_roles_seeded

from student import auth
from student.models import CourseEnrollment
from student.roles import CourseCreatorRole, CourseInstructorRole
from opaque_keys import InvalidKeyError


TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


class MongoCollectionFindWrapper(object):
    def __init__(self, original):
        self.original = original
        self.counter = 0

    def find(self, query, *args, **kwargs):
        self.counter = self.counter + 1
        return self.original(query, *args, **kwargs)


def get_url(handler_name, key_value, key_name='usage_key_string', kwargs=None):
    # Helper function for getting HTML for a page in Studio and
    # checking that it does not error.
    return reverse_url(handler_name, key_name, key_value, kwargs)


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
    """
    Tests that rely on the toy courses.
    TODO: refactor using CourseFactory so they do not.
    """
    def setUp(self):

        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True

        # Save the data that we've just changed to the db.
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def check_components_on_page(self, component_types, expected_types):
        """
        Ensure that the right types end up on the page.

        component_types is the list of advanced components.

        expected_types is the list of elements that should appear on the page.

        expected_types and component_types should be similar, but not
        exactly the same -- for example, 'video' in
        component_types should cause 'Video' to be present.
        """
        store = modulestore('direct')
        _, course_items = import_from_xml(store, 'common/test/data/', ['simple'])
        course = course_items[0]
        course.advanced_modules = component_types
        store.update_item(course, self.user.id)

        # just pick one vertical
        descriptor = store.get_items(course.id, category='vertical',)
        resp = self.client.get_html(get_url('unit_handler', descriptor[0].location))
        self.assertEqual(resp.status_code, 200)
        _test_no_locations(self, resp)

        for expected in expected_types:
            self.assertIn(expected, resp.content)

    def test_advanced_components_in_edit_unit(self):
        # This could be made better, but for now let's just assert that we see the advanced modules mentioned in the page
        # response HTML
        self.check_components_on_page(
            ADVANCED_COMPONENT_TYPES,
            ['Word cloud', 'Annotation', 'Text Annotation', 'Video Annotation', 'Image Annotation',
             'Open Response Assessment', 'Peer Grading Interface', 'openassessment'],
        )

    def test_advanced_components_require_two_clicks(self):
        self.check_components_on_page(['word_cloud'], ['Word cloud'])

    def test_malformed_edit_unit_request(self):
        store = modulestore('direct')
        _, course_items = import_from_xml(store, 'common/test/data/', ['simple'])

        # just pick one vertical
        usage_key = course_items[0].id.make_usage_key('vertical', None)

        resp = self.client.get_html(get_url('unit_handler', usage_key))
        self.assertEqual(resp.status_code, 400)
        _test_no_locations(self, resp, status_code=400)

    def check_edit_unit(self, test_course_name):
        _, course_items = import_from_xml(modulestore('direct'), 'common/test/data/', [test_course_name])

        items = modulestore().get_items(course_items[0].id, category='vertical')
        self._check_verticals(items)

    def _lock_an_asset(self, content_store, course_id):
        """
        Lock an arbitrary asset in the course
        :param course_location:
        """
        course_assets, __ = content_store.get_all_content_for_course(course_id)
        self.assertGreater(len(course_assets), 0, "No assets to lock")
        asset_id = course_assets[0]['_id']
        asset_key = StaticContent.compute_location(course_id, asset_id['name'])
        content_store.set_attr(asset_key, 'locked', True)
        return asset_key

    def test_edit_unit_toy(self):
        self.check_edit_unit('toy')

    def _get_draft_counts(self, item):
        cnt = 1 if getattr(item, 'is_draft', False) else 0
        for child in item.get_children():
            cnt = cnt + self._get_draft_counts(child)

        return cnt

    def test_get_items(self):
        '''
        This verifies a bug we had where the None setting in get_items() meant 'wildcard'
        Unfortunately, None = published for the revision field, so get_items() would return
        both draft and non-draft copies.
        '''
        direct_store = modulestore('direct')
        draft_store = modulestore('draft')
        _, course_items = import_from_xml(direct_store, 'common/test/data/', ['simple'])
        course_key = course_items[0].id
        html_usage_key = course_key.make_usage_key('html', 'test_html')

        html_module_from_draft_store = draft_store.get_item(html_usage_key)
        draft_store.convert_to_draft(html_module_from_draft_store.location)

        # Query get_items() and find the html item. This should just return back a single item (not 2).

        direct_store_items = direct_store.get_items(course_key)
        html_items_from_direct_store = [item for item in direct_store_items if (item.location == html_usage_key)]
        self.assertEqual(len(html_items_from_direct_store), 1)
        self.assertFalse(getattr(html_items_from_direct_store[0], 'is_draft', False))

        # Fetch from the draft store. Note that even though we pass
        # None in the revision field, the draft store will replace that with 'draft'.
        draft_store_items = draft_store.get_items(course_key)
        html_items_from_draft_store = [item for item in draft_store_items if (item.location == html_usage_key)]
        self.assertEqual(len(html_items_from_draft_store), 1)
        self.assertTrue(getattr(html_items_from_draft_store[0], 'is_draft', False))


    def test_draft_metadata(self):
        '''
        This verifies a bug we had where inherited metadata was getting written to the
        module as 'own-metadata' when publishing. Also verifies the metadata inheritance is
        properly computed
        '''
        store = modulestore('direct')
        draft_store = modulestore('draft')
        import_from_xml(store, 'common/test/data/', ['simple'])

        course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
        html_usage_key = course_key.make_usage_key('html', 'test_html')
        course = draft_store.get_course(course_key)
        html_module = draft_store.get_item(html_usage_key)

        self.assertEqual(html_module.graceperiod, course.graceperiod)
        self.assertNotIn('graceperiod', own_metadata(html_module))

        draft_store.convert_to_draft(html_module.location)

        # refetch to check metadata
        html_module = draft_store.get_item(html_usage_key)

        self.assertEqual(html_module.graceperiod, course.graceperiod)
        self.assertNotIn('graceperiod', own_metadata(html_module))

        # publish module
        draft_store.publish(html_module.location, 0)

        # refetch to check metadata
        html_module = draft_store.get_item(html_usage_key)

        self.assertEqual(html_module.graceperiod, course.graceperiod)
        self.assertNotIn('graceperiod', own_metadata(html_module))

        # put back in draft and change metadata and see if it's now marked as 'own_metadata'
        draft_store.convert_to_draft(html_module.location)
        html_module = draft_store.get_item(html_usage_key)

        new_graceperiod = timedelta(hours=1)

        self.assertNotIn('graceperiod', own_metadata(html_module))
        html_module.graceperiod = new_graceperiod
        # Save the data that we've just changed to the underlying
        # MongoKeyValueStore before we update the mongo datastore.
        html_module.save()
        self.assertIn('graceperiod', own_metadata(html_module))
        self.assertEqual(html_module.graceperiod, new_graceperiod)

        draft_store.update_item(html_module, self.user.id)

        # read back to make sure it reads as 'own-metadata'
        html_module = draft_store.get_item(html_usage_key)

        self.assertIn('graceperiod', own_metadata(html_module))
        self.assertEqual(html_module.graceperiod, new_graceperiod)

        # republish
        draft_store.publish(html_module.location, 0)

        # and re-read and verify 'own-metadata'
        draft_store.convert_to_draft(html_module.location)
        html_module = draft_store.get_item(html_usage_key)

        self.assertIn('graceperiod', own_metadata(html_module))
        self.assertEqual(html_module.graceperiod, new_graceperiod)

    def test_get_depth_with_drafts(self):
        import_from_xml(modulestore('direct'), 'common/test/data/', ['simple'])

        course_key = SlashSeparatedCourseKey('edX', 'simple', '2012_Fall')
        course = modulestore('draft').get_course(course_key)

        # make sure no draft items have been returned
        num_drafts = self._get_draft_counts(course)
        self.assertEqual(num_drafts, 0)

        problem_usage_key = course_key.make_usage_key('problem', 'ps01-simple')
        problem = modulestore('draft').get_item(problem_usage_key)

        # put into draft
        modulestore('draft').convert_to_draft(problem.location)

        # make sure we can query that item and verify that it is a draft
        draft_problem = modulestore('draft').get_item(problem_usage_key)
        self.assertTrue(getattr(draft_problem, 'is_draft', False))

        # now requery with depth
        course = modulestore('draft').get_course(course_key)

        # make sure just one draft item have been returned
        num_drafts = self._get_draft_counts(course)
        self.assertEqual(num_drafts, 1)

    def test_no_static_link_rewrites_on_import(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course = course_items[0]

        handouts_usage_key = course.id.make_usage_key('course_info', 'handouts')
        handouts = module_store.get_item(handouts_usage_key)
        self.assertIn('/static/', handouts.data)

        handouts_usage_key = course.id.make_usage_key('html', 'toyhtml')
        handouts = module_store.get_item(handouts_usage_key)
        self.assertIn('/static/', handouts.data)

    @mock.patch('xmodule.course_module.requests.get')
    def test_import_textbook_as_content_element(self, mock_get):
        mock_get.return_value.text = dedent("""
            <?xml version="1.0"?><table_of_contents>
            <entry page="5" page_label="ii" name="Table of Contents"/>
            </table_of_contents>
        """).strip()

        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))
        self.assertGreater(len(course.textbooks), 0)

    def test_import_polls(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_key = course_items[0].id

        items = module_store.get_items(course_key, category='poll_question')
        found = len(items) > 0

        self.assertTrue(found)
        # check that there's actually content in the 'question' field
        self.assertGreater(len(items[0].question), 0)

    def test_xlint_fails(self):
        err_cnt = perform_xlint('common/test/data', ['toy'])
        self.assertGreater(err_cnt, 0)

    @override_settings(COURSES_WITH_UNSAFE_CODE=['edX/toy/.*'])
    def test_module_preview_in_whitelist(self):
        """
        Tests the ajax callback to render an XModule
        """
        direct_store = modulestore('direct')
        _, course_items = import_from_xml(direct_store, 'common/test/data/', ['toy'])
        usage_key = course_items[0].id.make_usage_key('vertical', 'vertical_test')

        # also try a custom response which will trigger the 'is this course in whitelist' logic
        resp = self.client.get_json(
            get_url('xblock_view_handler', usage_key, kwargs={'view_name': 'container_preview'})
        )
        self.assertEqual(resp.status_code, 200)
        # TODO: uncomment when preview no longer has locations being returned.
        # _test_no_locations(self, resp)

        # These are the data-ids of the xblocks contained in the vertical.
        self.assertContains(resp, 'edX+toy+2012_Fall+video+sample_video')
        self.assertContains(resp, 'edX+toy+2012_Fall+video+separate_file_video')
        self.assertContains(resp, 'edX+toy+2012_Fall+video+video_with_end_time')
        self.assertContains(resp, 'edX+toy+2012_Fall+poll_question+T1_changemind_poll_foo_2')

    def test_delete(self):
        direct_store = modulestore('direct')
        course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')

        chapterloc = ItemFactory.create(parent_location=course.location, display_name="Chapter").location
        ItemFactory.create(parent_location=chapterloc, category='sequential', display_name="Sequential")

        sequential_key = course.id.make_usage_key('sequential', 'Sequential')
        sequential = direct_store.get_item(sequential_key)
        chapter_key = course.id.make_usage_key('chapter', 'Chapter')
        chapter = direct_store.get_item(chapter_key)

        # make sure the parent points to the child object which is to be deleted
        self.assertTrue(sequential.location in chapter.children)

        self.client.delete(get_url('xblock_handler', sequential_key), {'recurse': True, 'all_versions': True})

        found = False
        try:
            direct_store.get_item(sequential_key)
            found = True
        except ItemNotFoundError:
            pass

        self.assertFalse(found)

        chapter = direct_store.get_item(chapter_key)

        # make sure the parent no longer points to the child object which was deleted
        self.assertFalse(sequential.location in chapter.children)

    def test_about_overrides(self):
        '''
        This test case verifies that a course can use specialized override for about data, e.g. /about/Fall_2012/effort.html
        while there is a base definition in /about/effort.html
        '''
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_key = course_items[0].id
        effort = module_store.get_item(course_key.make_usage_key('about', 'effort'))
        self.assertEqual(effort.data, '6 hours')

        # this one should be in a non-override folder
        effort = module_store.get_item(course_key.make_usage_key('about', 'end_date'))
        self.assertEqual(effort.data, 'TBD')

    def test_asset_import(self):
        '''
        This test validates that an image asset is imported and a thumbnail was generated for a .gif
        '''
        content_store = contentstore()

        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, verbose=True)

        course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))

        self.assertIsNotNone(course)

        # make sure we have some assets in our contentstore
        all_assets, __ = content_store.get_all_content_for_course(course.id)
        self.assertGreater(len(all_assets), 0)

        # make sure we have some thumbnails in our contentstore
        content_store.get_all_content_thumbnails_for_course(course.id)

        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        #
        # self.assertGreater(len(all_thumbnails), 0)

        content = None
        try:
            location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
            content = content_store.find(location)
        except NotFoundError:
            pass

        self.assertIsNotNone(content)

        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        # self.assertIsNotNone(content.thumbnail_location)
        #
        # thumbnail = None
        # try:
        #    thumbnail = content_store.find(content.thumbnail_location)
        # except:
        #    pass
        #
        # self.assertIsNotNone(thumbnail)

    def test_asset_delete_and_restore(self):
        '''
        This test will exercise the soft delete/restore functionality of the assets
        '''
        content_store, trash_store, thumbnail_location, _location = self._delete_asset_in_course()
        asset_location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')

        # now try to find it in store, but they should not be there any longer
        content = content_store.find(asset_location, throw_on_not_found=False)
        self.assertIsNone(content)

        if thumbnail_location:
            thumbnail = content_store.find(thumbnail_location, throw_on_not_found=False)
            self.assertIsNone(thumbnail)

        # now try to find it and the thumbnail in trashcan - should be in there
        content = trash_store.find(asset_location, throw_on_not_found=False)
        self.assertIsNotNone(content)

        if thumbnail_location:
            thumbnail = trash_store.find(thumbnail_location, throw_on_not_found=False)
            self.assertIsNotNone(thumbnail)

        # let's restore the asset
        restore_asset_from_trashcan('/c4x/edX/toy/asset/sample_static.txt')

        # now try to find it in courseware store, and they should be back after restore
        content = content_store.find(asset_location, throw_on_not_found=False)
        self.assertIsNotNone(content)

        if thumbnail_location:
            thumbnail = content_store.find(thumbnail_location, throw_on_not_found=False)
            self.assertIsNotNone(thumbnail)

    def _delete_asset_in_course(self):
        """
        Helper method for:
          1) importing course from xml
          2) finding asset in course (verifying non-empty)
          3) computing thumbnail location of asset
          4) deleting the asset from the course
        """

        content_store = contentstore()
        trash_store = contentstore('trashcan')
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)

        # look up original (and thumbnail) in content store, should be there after import
        location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/sample_static.txt')
        content = content_store.find(location, throw_on_not_found=False)
        thumbnail_location = content.thumbnail_location
        self.assertIsNotNone(content)

        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        # self.assertIsNotNone(thumbnail_location)

        # go through the website to do the delete, since the soft-delete logic is in the view
        course = course_items[0]
        url = reverse_course_url(
            'assets_handler',
            course.id,
            kwargs={'asset_key_string': course.id.make_asset_key('asset', 'sample_static.txt')}
        )
        resp = self.client.delete(url)
        self.assertEqual(resp.status_code, 204)

        return content_store, trash_store, thumbnail_location, location

    def test_course_info_updates_import_export(self):
        """
        Test that course info updates are imported and exported with all content fields ('data', 'items')
        """
        content_store = contentstore()
        module_store = modulestore('direct')
        data_dir = "common/test/data/"
        import_from_xml(module_store, data_dir, ['course_info_updates'],
                        static_content_store=content_store, verbose=True)

        course_id = SlashSeparatedCourseKey('edX', 'course_info_updates', '2014_T1')
        course = module_store.get_course(course_id)

        self.assertIsNotNone(course)

        course_updates = module_store.get_item(course_id.make_usage_key('course_info', 'updates'))

        self.assertIsNotNone(course_updates)

        # check that course which is imported has files 'updates.html' and 'updates.items.json'
        filesystem = OSFS(data_dir + 'course_info_updates/info')
        self.assertTrue(filesystem.exists('updates.html'))
        self.assertTrue(filesystem.exists('updates.items.json'))

        # verify that course info update module has same data content as in data file from which it is imported
        # check 'data' field content
        with filesystem.open('updates.html', 'r') as course_policy:
            on_disk = course_policy.read()
            self.assertEqual(course_updates.data, on_disk)

        # check 'items' field content
        with filesystem.open('updates.items.json', 'r') as course_policy:
            on_disk = loads(course_policy.read())
            self.assertEqual(course_updates.items, on_disk)

        # now export the course to a tempdir and test that it contains files 'updates.html' and 'updates.items.json'
        # with same content as in course 'info' directory
        root_dir = path(mkdtemp_clean())
        print 'Exporting to tempdir = {0}'.format(root_dir)
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export')

        # check that exported course has files 'updates.html' and 'updates.items.json'
        filesystem = OSFS(root_dir / 'test_export/info')
        self.assertTrue(filesystem.exists('updates.html'))
        self.assertTrue(filesystem.exists('updates.items.json'))

        # verify that exported course has same data content as in course_info_update module
        with filesystem.open('updates.html', 'r') as grading_policy:
            on_disk = grading_policy.read()
            self.assertEqual(on_disk, course_updates.data)

        with filesystem.open('updates.items.json', 'r') as grading_policy:
            on_disk = loads(grading_policy.read())
            self.assertEqual(on_disk, course_updates.items)

    def test_empty_trashcan(self):
        '''
        This test will exercise the emptying of the asset trashcan
        '''
        __, trash_store, __, _location = self._delete_asset_in_course()

        # make sure there's something in the trashcan
        course_id = SlashSeparatedCourseKey('edX', 'toy', '6.002_Spring_2012')
        all_assets, __ = trash_store.get_all_content_for_course(course_id)
        self.assertGreater(len(all_assets), 0)

        # make sure we have some thumbnails in our trashcan
        _all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_id)
        #
        # cdodge: temporarily comment out assertion on thumbnails because many environments
        # will not have the jpeg converter installed and this test will fail
        #
        # self.assertGreater(len(all_thumbnails), 0)

        # empty the trashcan
        empty_asset_trashcan([course_id])

        # make sure trashcan is empty
        all_assets, count = trash_store.get_all_content_for_course(course_id)
        self.assertEqual(len(all_assets), 0)
        self.assertEqual(count, 0)

        all_thumbnails = trash_store.get_all_content_thumbnails_for_course(course_id)
        self.assertEqual(len(all_thumbnails), 0)

    def test_clone_course(self):

        course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
            'run': '2013_Spring',
        }

        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])

        source_course_id = course_items[0].id
        dest_course_id = _get_course_id(course_data)

        # get a vertical (and components in it) to put into 'draft'
        # this is to assert that draft content is also cloned over
        vertical = module_store.get_item(
            source_course_id.make_usage_key('vertical', 'vertical_test'),
            depth=1
        )

        draft_store.convert_to_draft(vertical.location)
        for child in vertical.get_children():
            draft_store.convert_to_draft(child.location)

        items = module_store.get_items(source_course_id, revision='draft')
        self.assertGreater(len(items), 0)

        _create_course(self, dest_course_id, course_data)

        content_store = contentstore()

        # now do the actual cloning
        clone_course(module_store, content_store, source_course_id, dest_course_id)

        # first assert that all draft content got cloned as well
        items = module_store.get_items(source_course_id, revision='draft')
        self.assertGreater(len(items), 0)
        clone_items = module_store.get_items(dest_course_id, revision='draft')
        self.assertGreater(len(clone_items), 0)
        self.assertEqual(len(items), len(clone_items))

        # now loop through all the units in the course and verify that the clone can render them, which
        # means the objects are at least present
        items = module_store.get_items(source_course_id, revision=None)
        self.assertGreater(len(items), 0)
        clone_items = module_store.get_items(dest_course_id, revision=None)
        self.assertGreater(len(clone_items), 0)

        for descriptor in items:
            source_item = module_store.get_item(descriptor.location)
            new_loc = descriptor.location.map_into_course(dest_course_id)
            print "Checking {0} should now also be at {1}".format(descriptor.location, new_loc)
            lookup_item = module_store.get_item(new_loc)

            if hasattr(source_item, 'data') and hasattr(lookup_item, 'data'):
                self.assertEqual(source_item.data, lookup_item.data)

            # also make sure that metadata was cloned over and filtered with own_metadata, i.e. inherited
            # values were not explicitly set
            self.assertEqual(own_metadata(source_item), own_metadata(lookup_item))

            # check that the children are as expected
            self.assertEqual(source_item.has_children, lookup_item.has_children)
            if source_item.has_children:
                expected_children = []
                for child_loc in source_item.children:
                    child_loc = child_loc.map_into_course(dest_course_id)
                    expected_children.append(child_loc)
                self.assertEqual(expected_children, lookup_item.children)

    def test_portable_link_rewrites_during_clone_course(self):
        course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
            'run': '2013_Spring'
        }

        module_store = modulestore('direct')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])

        source_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
        dest_course_id = _get_course_id(course_data)

        # let's force a non-portable link in the clone source
        # as a final check, make sure that any non-portable links are rewritten during cloning
        html_module = module_store.get_item(source_course_id.make_usage_key('html', 'nonportable'))

        self.assertIsInstance(html_module.data, basestring)
        new_data = html_module.data = html_module.data.replace('/static/', '/c4x/{0}/{1}/asset/'.format(
            source_course_id.org, source_course_id.run))
        module_store.update_item(html_module, self.user.id)

        html_module = module_store.get_item(html_module.location)
        self.assertEqual(new_data, html_module.data)

        # create the destination course
        _create_course(self, dest_course_id, course_data)

        # do the actual cloning
        clone_course(module_store, content_store, source_course_id, dest_course_id)

        # make sure that any non-portable links are rewritten during cloning
        html_module = module_store.get_item(dest_course_id.make_usage_key('html', 'nonportable'))

        self.assertIn('/asset/foo.jpg', html_module.data)

    def test_illegal_draft_crud_ops(self):
        draft_store = modulestore('draft')

        course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')

        location = course.id.make_usage_key('chapter', 'neuvo')
        # Ensure draft mongo store does not create drafts for things that shouldn't be draft
        newobject = draft_store.create_and_save_xmodule(location)
        self.assertFalse(getattr(newobject, 'is_draft', False))
        with self.assertRaises(InvalidVersionError):
            draft_store.convert_to_draft(location)
        chapter = draft_store.get_item(location)
        chapter.data = 'chapter data'

        draft_store.update_item(chapter, self.user.id)
        newobject = draft_store.get_item(chapter.location)
        self.assertFalse(getattr(newobject, 'is_draft', False))

        with self.assertRaises(InvalidVersionError):
            draft_store.unpublish(location)

    def test_bad_contentstore_request(self):
        resp = self.client.get_html('http://localhost:8001/c4x/CDX/123123/asset/&images_circuits_Lab7Solution2.png')
        self.assertEqual(resp.status_code, 400)
        _test_no_locations(self, resp, 400)

    def test_rewrite_nonportable_links_on_import(self):
        module_store = modulestore('direct')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)

        # first check a static asset link
        course_key = SlashSeparatedCourseKey('edX', 'toy', 'run')
        html_module_location = course_key.make_usage_key('html', 'nonportable')
        html_module = module_store.get_item(html_module_location)
        self.assertIn('/static/foo.jpg', html_module.data)

        # then check a intra courseware link
        html_module_location = course_key.make_usage_key('html', 'nonportable_link')
        html_module = module_store.get_item(html_module_location)
        self.assertIn('/jump_to_id/nonportable_link', html_module.data)

    def test_delete_course(self):
        """
        This test will import a course, make a draft item, and delete it. This will also assert that the
        draft content is also deleted
        """
        module_store = modulestore('direct')

        content_store = contentstore()
        draft_store = modulestore('draft')

        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)

        course_id = course_items[0].id

        # get a vertical (and components in it) to put into 'draft'
        vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)

        draft_store.convert_to_draft(vertical.location)
        for child in vertical.get_children():
            draft_store.convert_to_draft(child.location)

        # delete the course
        delete_course(module_store, content_store, course_id, commit=True)

        # assert that there's absolutely no non-draft modules in the course
        # this should also include all draft items
        items = module_store.get_items(course_id)
        self.assertEqual(len(items), 0)

        # assert that all content in the asset library is also deleted
        assets, count = content_store.get_all_content_for_course(course_id)
        self.assertEqual(len(assets), 0)
        self.assertEqual(count, 0)

    def verify_content_existence(self, store, root_dir, course_id, dirname, category_name, filename_suffix=''):
        filesystem = OSFS(root_dir / 'test_export')
        self.assertTrue(filesystem.exists(dirname))

        items = store.get_items(course_id, category=category_name)

        for item in items:
            filesystem = OSFS(root_dir / ('test_export/' + dirname))
            self.assertTrue(filesystem.exists(item.location.name + filename_suffix))

    @mock.patch('xmodule.course_module.requests.get')
    def test_export_course(self, mock_get):
        mock_get.return_value.text = dedent("""
            <?xml version="1.0"?><table_of_contents>
            <entry page="5" page_label="ii" name="Table of Contents"/>
            </table_of_contents>
        """).strip()

        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store)
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        # get a vertical (and components in it) to copy into an orphan sub dag
        vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
        # We had a bug where orphaned draft nodes caused export to fail. This is here to cover that case.
        vertical.location = mongo.draft.as_draft(vertical.location.replace(name='no_references'))

        draft_store.update_item(vertical, allow_not_found=True)
        orphan_vertical = draft_store.get_item(vertical.location)
        self.assertEqual(orphan_vertical.location.name, 'no_references')

        # get the original vertical (and components in it) to put into 'draft'
        vertical = module_store.get_item(course_id.make_usage_key('vertical', 'vertical_test'), depth=1)
        self.assertEqual(len(orphan_vertical.children), len(vertical.children))
        draft_store.convert_to_draft(vertical.location)
        for child in vertical.get_children():
            draft_store.convert_to_draft(child.location)

        root_dir = path(mkdtemp_clean())

        # now create a new/different private (draft only) vertical
        vertical.location = mongo.draft.as_draft(course_id.make_usage_key('vertical', 'a_private_vertical'))
        draft_store.update_item(vertical, allow_not_found=True)
        private_vertical = draft_store.get_item(vertical.location)
        vertical = None  # blank out b/c i destructively manipulated its location 2 lines above

        # add the new private to list of children
        sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))
        private_location_no_draft = private_vertical.location.replace(revision=None)
        sequential.children.append(private_location_no_draft)
        module_store.update_item(sequential, self.user.id)

        # read back the sequential, to make sure we have a pointer to
        sequential = module_store.get_item(course_id.make_usage_key('sequential', 'vertical_sequential'))

        self.assertIn(private_location_no_draft, sequential.children)

        locked_asset_key = self._lock_an_asset(content_store, course_id)
        locked_asset_attrs = content_store.get_attrs(locked_asset_key)
        # the later import will reupload
        del locked_asset_attrs['uploadDate']

        print 'Exporting to tempdir = {0}'.format(root_dir)

        # export out to a tempdir
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)

        # check for static tabs
        self.verify_content_existence(module_store, root_dir, course_id, 'tabs', 'static_tab', '.html')

        # check for about content
        self.verify_content_existence(module_store, root_dir, course_id, 'about', 'about', '.html')

        # check for grading_policy.json
        filesystem = OSFS(root_dir / 'test_export/policies/2012_Fall')
        self.assertTrue(filesystem.exists('grading_policy.json'))

        course = module_store.get_course(course_id)
        # compare what's on disk compared to what we have in our course
        with filesystem.open('grading_policy.json', 'r') as grading_policy:
            on_disk = loads(grading_policy.read())
            self.assertEqual(on_disk, course.grading_policy)

        # check for policy.json
        self.assertTrue(filesystem.exists('policy.json'))

        # compare what's on disk to what we have in the course module
        with filesystem.open('policy.json', 'r') as course_policy:
            on_disk = loads(course_policy.read())
            self.assertIn('course/2012_Fall', on_disk)
            self.assertEqual(on_disk['course/2012_Fall'], own_metadata(course))

        # remove old course
        delete_course(module_store, content_store, course_id, commit=True)
        # reimport over old course
        self.check_import(
            module_store, root_dir, draft_store, content_store, course_id,
            locked_asset_key, locked_asset_attrs
        )
        # import to different course id
        self.check_import(
            module_store, root_dir, draft_store, content_store, SlashSeparatedCourseKey('anotherX', 'anotherToy', 'Someday'),
            locked_asset_key, locked_asset_attrs
        )

        shutil.rmtree(root_dir)

    def check_import(self, module_store, root_dir, draft_store, content_store, course_id,
                     locked_asset_key, locked_asset_attrs):
        # reimport
        import_from_xml(
            module_store,
            root_dir,
            ['test_export'],
            draft_store=draft_store,
            static_content_store=content_store,
            target_course_id=course_id,
        )

        items = module_store.get_items(course_id, category='vertical')
        self._check_verticals(items)

        # verify that we have the content in the draft store as well
        vertical = draft_store.get_item(
            course_id.make_usage_key('vertical', 'vertical_test'),
            depth=1
        )

        self.assertTrue(getattr(vertical, 'is_draft', False))
        self.assertNotIn('index_in_children_list', vertical.xml_attributes)
        self.assertNotIn('parent_sequential_url', vertical.xml_attributes)

        for child in vertical.get_children():
            self.assertTrue(getattr(child, 'is_draft', False))
            self.assertNotIn('index_in_children_list', child.xml_attributes)
            if hasattr(child, 'data'):
                self.assertNotIn('index_in_children_list', child.data)
            self.assertNotIn('parent_sequential_url', child.xml_attributes)
            if hasattr(child, 'data'):
                self.assertNotIn('parent_sequential_url', child.data)

        # make sure that we don't have a sequential that is in draft mode
        sequential = draft_store.get_item(
            course_id.make_usage_key('sequential', 'vertical_sequential')
        )

        self.assertFalse(getattr(sequential, 'is_draft', False))

        # verify that we have the private vertical
        test_private_vertical = draft_store.get_item(
            course_id.make_usage_key('vertical', 'a_private_vertical')
        )

        self.assertTrue(getattr(test_private_vertical, 'is_draft', False))

        # make sure the textbook survived the export/import
        course = module_store.get_course(course_id)

        self.assertGreater(len(course.textbooks), 0)

        locked_asset_key = locked_asset_key.map_into_course(course_id)
        new_attrs = content_store.get_attrs(locked_asset_key)
        for key, value in locked_asset_attrs.iteritems():
            if key == '_id':
                self.assertEqual(value['name'], new_attrs[key]['name'])
            elif key == 'filename':
                pass
            else:
                self.assertEqual(value, new_attrs[key])

    def test_export_course_with_metadata_only_video(self):
        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        # create a new video module and add it as a child to a vertical
        # this re-creates a bug whereby since the video template doesn't have
        # anything in 'data' field, the export was blowing up
        verticals = module_store.get_items(course_id, category='vertical')

        self.assertGreater(len(verticals), 0)

        parent = verticals[0]

        ItemFactory.create(parent_location=parent.location, category="video", display_name="untitled")

        root_dir = path(mkdtemp_clean())

        print 'Exporting to tempdir = {0}'.format(root_dir)

        # export out to a tempdir
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)

        shutil.rmtree(root_dir)

    def test_export_course_with_metadata_only_word_cloud(self):
        """
        Similar to `test_export_course_with_metadata_only_video`.
        """
        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['word_cloud'])
        course_id = SlashSeparatedCourseKey('HarvardX', 'ER22x', '2013_Spring')

        verticals = module_store.get_items(course_id, category='vertical')

        self.assertGreater(len(verticals), 0)

        parent = verticals[0]

        ItemFactory.create(parent_location=parent.location, category="word_cloud", display_name="untitled")

        root_dir = path(mkdtemp_clean())

        print 'Exporting to tempdir = {0}'.format(root_dir)

        # export out to a tempdir
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_export', draft_modulestore=draft_store)

        shutil.rmtree(root_dir)

    def test_empty_data_roundtrip(self):
        """
        Test that an empty `data` field is preserved through
        export/import.
        """
        module_store = modulestore('direct')
        draft_store = modulestore('draft')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        verticals = module_store.get_items(course_id, category='vertical')

        self.assertGreater(len(verticals), 0)

        parent = verticals[0]

        # Create a module, and ensure that its `data` field is empty
        word_cloud = ItemFactory.create(parent_location=parent.location, category="word_cloud", display_name="untitled")
        del word_cloud.data
        self.assertEquals(word_cloud.data, '')

        # Export the course
        root_dir = path(mkdtemp_clean())
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip', draft_modulestore=draft_store)

        # Reimport and get the video back
        import_from_xml(module_store, root_dir)
        imported_word_cloud = module_store.get_item(course_id.make_usage_key('word_cloud', 'untitled'))

        # It should now contain empty data
        self.assertEquals(imported_word_cloud.data, '')

    def test_html_export_roundtrip(self):
        """
        Test that a course which has HTML that has style formatting is preserved in export/import
        """
        module_store = modulestore('direct')
        content_store = contentstore()

        import_from_xml(module_store, 'common/test/data/', ['toy'])

        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        # Export the course
        root_dir = path(mkdtemp_clean())
        export_to_xml(module_store, content_store, course_id, root_dir, 'test_roundtrip')

        # Reimport and get the video back
        import_from_xml(module_store, root_dir)

        # get the sample HTML with styling information
        html_module = module_store.get_item(course_id.make_usage_key('html', 'with_styling'))
        self.assertIn('<p style="font:italic bold 72px/30px Georgia, serif; color: red; ">', html_module.data)

        # get the sample HTML with just a simple <img> tag information
        html_module = module_store.get_item(course_id.make_usage_key('html', 'just_img'))
        self.assertIn('<img src="/static/foo_bar.jpg" />', html_module.data)

    def test_course_handouts_rewrites(self):
        module_store = modulestore('direct')

        # import a test course
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = course_items[0].id

        handouts_location = course_id.make_usage_key('course_info', 'handouts')

        # get module info (json)
        resp = self.client.get(get_url('xblock_handler', handouts_location))

        # make sure we got a successful response
        self.assertEqual(resp.status_code, 200)
        # check that /static/ has been converted to the full path
        # note, we know the link it should be because that's what in the 'toy' course in the test data
        self.assertContains(resp, '/c4x/edX/toy/asset/handouts_sample_handout.txt')

    def test_prefetch_children(self):
        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        wrapper = MongoCollectionFindWrapper(module_store.collection.find)
        module_store.collection.find = wrapper.find
        print module_store.metadata_inheritance_cache_subsystem
        print module_store.request_cache
        course = module_store.get_course(course_id, depth=2)

        # make sure we haven't done too many round trips to DB
        # note we say 3 round trips here for 1) the course, and 2 & 3) for the chapters and sequentials
        # Because we're querying from the top of the tree, we cache information needed for inheritance,
        # so we don't need to make an extra query to compute it.
        self.assertEqual(wrapper.counter, 3)

        # make sure we pre-fetched a known sequential which should be at depth=2
        self.assertTrue(course_id.make_usage_key('sequential', 'vertical_sequential') in course.system.module_data)

        # make sure we don't have a specific vertical which should be at depth=3
        self.assertFalse(course_id.make_usage_key('vertical', 'vertical_test') in course.system.module_data)

    def test_export_course_without_content_store(self):
        module_store = modulestore('direct')
        content_store = contentstore()

        # Create toy course

        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])
        course_id = course_items[0].id

        root_dir = path(mkdtemp_clean())

        print 'Exporting to tempdir = {0}'.format(root_dir)
        export_to_xml(module_store, None, course_id, root_dir, 'test_export_no_content_store')

        # Delete the course from module store and reimport it

        delete_course(module_store, content_store, course_id, commit=True)

        import_from_xml(
            module_store, root_dir, ['test_export_no_content_store'],
            draft_store=None,
            static_content_store=None,
            target_course_id=course_id
        )

        # Verify reimported course

        items = module_store.get_items(
            course_id,
            category='sequential',
            name='vertical_sequential'
        )
        self.assertEqual(len(items), 1)

    def _check_verticals(self, items):
        """ Test getting the editing HTML for each vertical. """
        # Assert is here to make sure that the course being tested actually has verticals (units) to check.
        self.assertGreater(len(items), 0)
        for descriptor in items:
            resp = self.client.get_html(get_url('unit_handler', descriptor.location))
            self.assertEqual(resp.status_code, 200)
            _test_no_locations(self, resp)


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreTest(ModuleStoreTestCase):
    """
    Tests for the CMS ContentStore application.
    """
    def setUp(self):
        """
        These tests need a user in the DB so that the django Test Client
        can log them in.
        They inherit from the ModuleStoreTestCase class so that the mongodb collection
        will be cleared out before each test case execution and deleted
        afterwards.
        """
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

        self.course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
            'run': '2013_Spring'
        }

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def assert_created_course(self, number_suffix=None):
        """
        Checks that the course was created properly.
        """
        test_course_data = {}
        test_course_data.update(self.course_data)
        if number_suffix:
            test_course_data['number'] = '{0}_{1}'.format(test_course_data['number'], number_suffix)
        course_key = _get_course_id(test_course_data)
        _create_course(self, course_key, test_course_data)
        # Verify that the creator is now registered in the course.
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_key))
        return test_course_data

    def assert_create_course_failed(self, error_message):
        """
        Checks that the course not created.
        """
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 400)
        data = parse_json(resp)
        self.assertEqual(data['error'], error_message)

    def test_create_course(self):
        """Test new course creation - happy path"""
        self.assert_created_course()

    def test_create_course_with_dots(self):
        """Test new course creation with dots in the name"""
        self.course_data['org'] = 'org.foo.bar'
        self.course_data['number'] = 'course.number'
        self.course_data['run'] = 'run.name'
        self.assert_created_course()

    def test_create_course_check_forum_seeding(self):
        """Test new course creation and verify forum seeding """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        self.assertTrue(are_permissions_roles_seeded(_get_course_id(test_course_data)))

    def test_forum_unseeding_on_delete(self):
        """Test new course creation and verify forum unseeding """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        course_id = _get_course_id(test_course_data)
        self.assertTrue(are_permissions_roles_seeded(course_id))
        delete_course_and_groups(course_id, commit=True)
        # should raise an exception for checking permissions on deleted course
        with self.assertRaises(ItemNotFoundError):
            are_permissions_roles_seeded(course_id)

    def test_forum_unseeding_with_multiple_courses(self):
        """Test new course creation and verify forum unseeding when there are multiple courses"""
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        second_course_data = self.assert_created_course(number_suffix=uuid4().hex)

        # unseed the forums for the first course
        course_id = _get_course_id(test_course_data)
        delete_course_and_groups(course_id, commit=True)
        # should raise an exception for checking permissions on deleted course
        with self.assertRaises(ItemNotFoundError):
            are_permissions_roles_seeded(course_id)

        second_course_id = _get_course_id(second_course_data)
        # permissions should still be there for the other course
        self.assertTrue(are_permissions_roles_seeded(second_course_id))

    def test_course_enrollments_and_roles_on_delete(self):
        """
        Test that course deletion doesn't remove course enrollments or user's roles
        """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        course_id = _get_course_id(test_course_data)

        # test that a user gets his enrollment and its 'student' role as default on creating a course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_id))
        self.assertTrue(self.user.roles.filter(name="Student", course_id=course_id))  # pylint: disable=no-member

        delete_course_and_groups(course_id, commit=True)
        # check that user's enrollment for this course is not deleted
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_id))
        # check that user has form role "Student" for this course even after deleting it
        self.assertTrue(self.user.roles.filter(name="Student", course_id=course_id))  # pylint: disable=no-member

    def test_course_access_groups_on_delete(self):
        """
        Test that course deletion removes users from 'instructor' and 'staff' groups of this course
        of all format e.g, 'instructor_edX/Course/Run', 'instructor_edX.Course.Run', 'instructor_Course'
        """
        test_course_data = self.assert_created_course(number_suffix=uuid4().hex)
        course_id = _get_course_id(test_course_data)

        # Add user in possible groups and check that user in instructor groups of this course
        instructor_role = CourseInstructorRole(course_id)

        auth.add_users(self.user, instructor_role, self.user)

        self.assertTrue(len(instructor_role.users_with_role()) > 0)

        # Now delete course and check that user not in instructor groups of this course
        delete_course_and_groups(course_id, commit=True)

        # Update our cached user since its roles have changed
        self.user = User.objects.get_by_natural_key(self.user.natural_key()[0])

        self.assertFalse(instructor_role.has_user(self.user))
        self.assertEqual(len(instructor_role.users_with_role()), 0)

    def test_create_course_duplicate_course(self):
        """Test new course creation - error path"""
        self.client.ajax_post('/course/', self.course_data)
        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')

    def assert_course_creation_failed(self, error_message):
        """
        Checks that the course did not get created
        """
        test_enrollment = False
        try:
            course_id = _get_course_id(self.course_data)
            initially_enrolled = CourseEnrollment.is_enrolled(self.user, course_id)
            test_enrollment = True
        except InvalidKeyError:
            # b/c the intent of the test with bad chars isn't to test auth but to test the handler, ignore
            pass
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertRegexpMatches(data['ErrMsg'], error_message)
        if test_enrollment:
            # One test case involves trying to create the same course twice. Hence for that course,
            # the user will be enrolled. In the other cases, initially_enrolled will be False.
            self.assertEqual(initially_enrolled, CourseEnrollment.is_enrolled(self.user, course_id))

    def test_create_course_duplicate_number(self):
        """Test new course creation - error path"""
        self.client.ajax_post('/course/', self.course_data)
        self.course_data['display_name'] = 'Robot Super Course Two'
        self.course_data['run'] = '2013_Summer'

        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')

    def test_create_course_case_change(self):
        """Test new course creation - error path due to case insensitive name equality"""
        self.course_data['number'] = 'capital'
        self.client.ajax_post('/course/', self.course_data)
        cache_current = self.course_data['org']
        self.course_data['org'] = self.course_data['org'].lower()
        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')
        self.course_data['org'] = cache_current

        self.client.ajax_post('/course/', self.course_data)
        cache_current = self.course_data['number']
        self.course_data['number'] = self.course_data['number'].upper()
        self.assert_course_creation_failed('There is already a course defined with the same organization, course number, and course run. Please change either organization or course number to be unique.')

    def test_course_substring(self):
        """
        Test that a new course can be created whose name is a substring of an existing course
        """
        self.client.ajax_post('/course/', self.course_data)
        cache_current = self.course_data['number']
        self.course_data['number'] = '{}a'.format(self.course_data['number'])
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 200)
        self.course_data['number'] = cache_current
        self.course_data['org'] = 'a{}'.format(self.course_data['org'])
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 200)

    def test_create_course_with_bad_organization(self):
        """Test new course creation - error path for bad organization name"""
        self.course_data['org'] = 'University of California, Berkeley'
        self.assert_course_creation_failed(
            r"(?s)Unable to create course 'Robot Super Course'.*: Invalid characters in u'University of California, Berkeley'")

    def test_create_course_with_course_creation_disabled_staff(self):
        """Test new course creation -- course creation disabled, but staff access."""
        with mock.patch.dict('django.conf.settings.FEATURES', {'DISABLE_COURSE_CREATION': True}):
            self.assert_created_course()

    def test_create_course_with_course_creation_disabled_not_staff(self):
        """Test new course creation -- error path for course creation disabled, not staff access."""
        with mock.patch.dict('django.conf.settings.FEATURES', {'DISABLE_COURSE_CREATION': True}):
            self.user.is_staff = False
            self.user.save()
            self.assert_course_permission_denied()

    def test_create_course_no_course_creators_staff(self):
        """Test new course creation -- course creation group enabled, staff, group is empty."""
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}):
            self.assert_created_course()

    def test_create_course_no_course_creators_not_staff(self):
        """Test new course creation -- error path for course creator group enabled, not staff, group is empty."""
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            self.user.is_staff = False
            self.user.save()
            self.assert_course_permission_denied()

    def test_create_course_with_course_creator(self):
        """Test new course creation -- use course creator group"""
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            auth.add_users(self.user, CourseCreatorRole(), self.user)
            self.assert_created_course()

    def test_create_course_with_unicode_in_id_disabled(self):
        """
        Test new course creation with feature setting: ALLOW_UNICODE_COURSE_ID disabled.
        """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ALLOW_UNICODE_COURSE_ID': False}):
            error_message = "Special characters not allowed in organization, course number, and course run."
            self.course_data['org'] = u'��������������'
            self.assert_create_course_failed(error_message)

            self.course_data['number'] = u'��chantillon'
            self.assert_create_course_failed(error_message)

            self.course_data['run'] = u'����������'
            self.assert_create_course_failed(error_message)

    def assert_course_permission_denied(self):
        """
        Checks that the course did not get created due to a PermissionError.
        """
        resp = self.client.ajax_post('/course/', self.course_data)
        self.assertEqual(resp.status_code, 403)

    def test_course_index_view_with_no_courses(self):
        """Test viewing the index page with no courses"""
        # Create a course so there is something to view
        resp = self.client.get_html('/course/')
        self.assertContains(
            resp,
            '<h1 class="page-header">My Courses</h1>',
            status_code=200,
            html=True
        )
        _test_no_locations(self, resp)

    def test_course_factory(self):
        """Test that the course factory works correctly."""
        course = CourseFactory.create()
        self.assertIsInstance(course, CourseDescriptor)

    def test_item_factory(self):
        """Test that the item factory works correctly."""
        course = CourseFactory.create()
        item = ItemFactory.create(parent_location=course.location)
        self.assertIsInstance(item, SequenceDescriptor)

    def test_course_index_view_with_course(self):
        """Test viewing the index page with an existing course"""
        CourseFactory.create(display_name='Robot Super Educational Course')
        resp = self.client.get_html('/course/')
        self.assertContains(
            resp,
            '<h3 class="course-title">Robot Super Educational Course</h3>',
            status_code=200,
            html=True
        )
        _test_no_locations(self, resp)

    def test_course_overview_view_with_course(self):
        """Test viewing the course overview page with an existing course"""
        course = CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')
        resp = self._show_course_overview(course.id)
        self.assertContains(
            resp,
            '<article class="courseware-overview" data-locator="location:MITx+999+Robot_Super_Course+course+Robot_Super_Course" data-course-key="slashes:MITx+999+Robot_Super_Course">',
            status_code=200,
            html=True
        )

    def test_create_item(self):
        """Test creating a new xblock instance."""
        course = _course_factory_create_course()

        section_data = {
            'parent_locator': unicode(course.location),
            'category': 'chapter',
            'display_name': 'Section One',
        }

        resp = self.client.ajax_post(reverse_url('xblock_handler'), section_data)
        _test_no_locations(self, resp, html=False)

        self.assertEqual(resp.status_code, 200)
        data = parse_json(resp)
        self.assertRegexpMatches(
            data['locator'],
            r"location:MITx\+999\+Robot_Super_Course\+chapter\+([0-9]|[a-f]){3,}$"
        )

    def test_capa_module(self):
        """Test that a problem treats markdown specially."""
        course = _course_factory_create_course()

        problem_data = {
            'parent_locator': unicode(course.location),
            'category': 'problem'
        }

        resp = self.client.ajax_post(reverse_url('xblock_handler'), problem_data)
        self.assertEqual(resp.status_code, 200)
        payload = parse_json(resp)
        problem_loc = UsageKey.from_string(payload['locator'])
        problem = get_modulestore(problem_loc).get_item(problem_loc)
        # should be a CapaDescriptor
        self.assertIsInstance(problem, CapaDescriptor, "New problem is not a CapaDescriptor")
        context = problem.get_context()
        self.assertIn('markdown', context, "markdown is missing from context")
        self.assertNotIn('markdown', problem.editable_metadata_fields, "Markdown slipped into the editable metadata fields")

    def test_cms_imported_course_walkthrough(self):
        """
        Import and walk through some common URL endpoints. This just verifies non-500 and no other
        correct behavior, so it is not a deep test
        """
        def test_get_html(handler):
            # Helper function for getting HTML for a page in Studio and
            # checking that it does not error.
            resp = self.client.get_html(
                get_url(handler, course_key, 'course_key_string')
            )
            self.assertEqual(resp.status_code, 200)
            _test_no_locations(self, resp)

        _, course_items = import_from_xml(modulestore('direct'), 'common/test/data/', ['simple'])
        course_key = course_items[0].id

        resp = self._show_course_overview(course_key)
        self.assertEqual(resp.status_code, 200)
        self.assertContains(resp, 'Chapter 2')

        # go to various pages
        test_get_html('import_handler')
        test_get_html('export_handler')
        test_get_html('course_team_handler')
        test_get_html('course_info_handler')
        test_get_html('checklists_handler')
        test_get_html('assets_handler')
        test_get_html('tabs_handler')
        test_get_html('settings_handler')
        test_get_html('grading_handler')
        test_get_html('advanced_settings_handler')
        test_get_html('textbooks_list_handler')

        # go look at a subsection page
        subsection_key = course_key.make_usage_key('sequential', 'test_sequence')
        resp = self.client.get_html(get_url('subsection_handler', subsection_key))
        self.assertEqual(resp.status_code, 200)
        _test_no_locations(self, resp)

        # go look at the Edit page
        unit_key = course_key.make_usage_key('vertical', 'test_vertical')
        resp = self.client.get_html(get_url('unit_handler', unit_key))
        self.assertEqual(resp.status_code, 200)
        _test_no_locations(self, resp)

        def delete_item(category, name):
            """ Helper method for testing the deletion of an xblock item. """
            item_key = course_key.make_usage_key(category, name)
            resp = self.client.delete(get_url('xblock_handler', item_key))
            self.assertEqual(resp.status_code, 204)
            _test_no_locations(self, resp, status_code=204, html=False)

        # delete a component
        delete_item(category='html', name='test_html')

        # delete a unit
        delete_item(category='vertical', name='test_vertical')

        # delete a unit
        delete_item(category='sequential', name='test_sequence')

        # delete a chapter
        delete_item(category='chapter', name='chapter_2')

    def test_import_into_new_course_id(self):
        module_store = modulestore('direct')
        target_course_id = _get_course_id(self.course_data)
        _create_course(self, target_course_id, self.course_data)

        import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)

        modules = module_store.get_items(target_course_id)

        # we should have a number of modules in there
        # we can't specify an exact number since it'll always be changing
        self.assertGreater(len(modules), 10)

        #
        # test various re-namespacing elements
        #

        # first check PDF textbooks, to make sure the url paths got updated
        course_module = module_store.get_course(target_course_id)

        self.assertEqual(len(course_module.pdf_textbooks), 1)
        self.assertEqual(len(course_module.pdf_textbooks[0]["chapters"]), 2)
        self.assertEqual(course_module.pdf_textbooks[0]["chapters"][0]["url"], '/static/Chapter1.pdf')
        self.assertEqual(course_module.pdf_textbooks[0]["chapters"][1]["url"], '/static/Chapter2.pdf')

    def test_import_into_new_course_id_wiki_slug_renamespacing(self):
        module_store = modulestore('direct')

        # If reimporting into the same course do not change the wiki_slug.
        target_course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
        course_data = {
            'org': target_course_id.org,
            'number': target_course_id.course,
            'display_name': 'Robot Super Course',
            'run': target_course_id.run
        }
        _create_course(self, target_course_id, course_data)
        course_module = module_store.get_course(target_course_id)
        course_module.wiki_slug = 'toy'
        course_module.save()

        # Import a course with wiki_slug == location.course
        import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
        course_module = module_store.get_course(target_course_id)
        self.assertEquals(course_module.wiki_slug, 'toy')

        # But change the wiki_slug if it is a different course.
        target_course_id = SlashSeparatedCourseKey('MITx', '999', '2013_Spring')
        course_data = {
            'org': target_course_id.org,
            'number': target_course_id.course,
            'display_name': 'Robot Super Course',
            'run': target_course_id.run
        }
        _create_course(self, target_course_id, course_data)

        # Import a course with wiki_slug == location.course
        import_from_xml(module_store, 'common/test/data/', ['toy'], target_course_id=target_course_id)
        course_module = module_store.get_course(target_course_id)
        self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')

        # Now try importing a course with wiki_slug == '{0}.{1}.{2}'.format(location.org, location.course, location.run)
        import_from_xml(module_store, 'common/test/data/', ['two_toys'], target_course_id=target_course_id)
        course_module = module_store.get_course(target_course_id)
        self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')

    def test_import_metadata_with_attempts_empty_string(self):
        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['simple'])
        did_load_item = False
        try:
            course_key = SlashSeparatedCourseKey('edX', 'simple', 'problem')
            usage_key = course_key.make_usage_key('problem', 'ps01-simple')
            module_store.get_item(usage_key)
            did_load_item = True
        except ItemNotFoundError:
            pass

        # make sure we found the item (e.g. it didn't error while loading)
        self.assertTrue(did_load_item)

    def test_forum_id_generation(self):
        module_store = modulestore('direct')
        course = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
        new_component_location = course.id.make_usage_key('discussion', 'new_component')

        # crate a new module and add it as a child to a vertical
        module_store.create_and_save_xmodule(new_component_location)

        new_discussion_item = module_store.get_item(new_component_location)

        self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$')

    def test_metadata_inheritance(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(module_store, 'common/test/data/', ['toy'])

        course = course_items[0]
        verticals = module_store.get_items(course.id, category='vertical')

        # let's assert on the metadata_inheritance on an existing vertical
        for vertical in verticals:
            self.assertEqual(course.xqa_key, vertical.xqa_key)
            self.assertEqual(course.start, vertical.start)

        self.assertGreater(len(verticals), 0)

        new_component_location = course.id.make_usage_key('html', 'new_component')

        # crate a new module and add it as a child to a vertical
        module_store.create_and_save_xmodule(new_component_location)
        parent = verticals[0]
        parent.children.append(new_component_location)
        module_store.update_item(parent, self.user.id)

        # flush the cache
        module_store.refresh_cached_metadata_inheritance_tree(new_component_location.course_key)
        new_module = module_store.get_item(new_component_location)

        # check for grace period definition which should be defined at the course level
        self.assertEqual(parent.graceperiod, new_module.graceperiod)
        self.assertEqual(parent.start, new_module.start)
        self.assertEqual(course.start, new_module.start)

        self.assertEqual(course.xqa_key, new_module.xqa_key)

        #
        # now let's define an override at the leaf node level
        #
        new_module.graceperiod = timedelta(1)
        module_store.update_item(new_module, self.user.id)

        # flush the cache and refetch
        module_store.refresh_cached_metadata_inheritance_tree(new_component_location.course_key)
        new_module = module_store.get_item(new_component_location)

        self.assertEqual(timedelta(1), new_module.graceperiod)

    def test_default_metadata_inheritance(self):
        course = CourseFactory.create()
        vertical = ItemFactory.create(parent_location=course.location)
        course.children.append(vertical)
        # in memory
        self.assertIsNotNone(course.start)
        self.assertEqual(course.start, vertical.start)
        self.assertEqual(course.textbooks, [])
        self.assertIn('GRADER', course.grading_policy)
        self.assertIn('GRADE_CUTOFFS', course.grading_policy)
        self.assertGreaterEqual(len(course.checklists), 4)

        # by fetching
        module_store = modulestore('direct')
        fetched_course = module_store.get_item(course.location)
        fetched_item = module_store.get_item(vertical.location)
        self.assertIsNotNone(fetched_course.start)
        self.assertEqual(course.start, fetched_course.start)
        self.assertEqual(fetched_course.start, fetched_item.start)
        self.assertEqual(course.textbooks, fetched_course.textbooks)
        # is this test too strict? i.e., it requires the dicts to be ==
        self.assertEqual(course.checklists, fetched_course.checklists)

    def test_image_import(self):
        """Test backwards compatibilty of course image."""
        module_store = modulestore('direct')

        content_store = contentstore()

        # Use conditional_and_poll, as it's got an image already
        import_from_xml(
            module_store,
            'common/test/data/',
            ['conditional_and_poll'],
            static_content_store=content_store
        )

        course = module_store.get_courses()[0]

        # Make sure the course image is set to the right place
        self.assertEqual(course.course_image, 'images_course_image.jpg')

        # Ensure that the imported course image is present -- this shouldn't raise an exception
        asset_key = course.id.make_asset_key('asset', course.course_image)
        content_store.find(asset_key)

    def _show_course_overview(self, course_key):
        """
        Show the course overview page.
        """
        resp = self.client.get_html(get_url('course_handler', course_key, 'course_key_string'))
        _test_no_locations(self, resp)
        return resp

    def test_wiki_slug(self):
        """When creating a course a unique wiki_slug should be set."""

        course_key = _get_course_id(self.course_data)
        _create_course(self, course_key, self.course_data)
        course_module = modulestore('direct').get_course(course_key)
        self.assertEquals(course_module.wiki_slug, 'MITx.999.2013_Spring')


@override_settings(MODULESTORE=TEST_MODULESTORE)
class MetadataSaveTestCase(ModuleStoreTestCase):
    """Test that metadata is correctly cached and decached."""

    def setUp(self):
        course = CourseFactory.create(
            org='edX', course='999', display_name='Robot Super Course')

        video_sample_xml = '''
        <video display_name="Test Video"
                youtube="1.0:p2Q6BrNhdh8,0.75:izygArpw-Qo,1.25:1EeWXzPdhSA,1.5:rABDYkeK0x8"
                show_captions="false"
                from="00:00:01"
                to="00:01:00">
            <source src="http://www.example.com/file.mp4"/>
            <track src="http://www.example.com/track"/>
        </video>
        '''
        self.video_descriptor = ItemFactory.create(
            parent_location=course.location, category='video',
            data={'data': video_sample_xml}
        )

    def test_metadata_not_persistence(self):
        """
        Test that descriptors which set metadata fields in their
        constructor are correctly deleted.
        """
        self.assertIn('html5_sources', own_metadata(self.video_descriptor))
        attrs_to_strip = {
            'show_captions',
            'youtube_id_1_0',
            'youtube_id_0_75',
            'youtube_id_1_25',
            'youtube_id_1_5',
            'start_time',
            'end_time',
            'source',
            'html5_sources',
            'track'
        }

        location = self.video_descriptor.location

        for field_name in attrs_to_strip:
            delattr(self.video_descriptor, field_name)

        self.assertNotIn('html5_sources', own_metadata(self.video_descriptor))
        get_modulestore(location).update_item(self.video_descriptor, '**replace_user**')
        module = get_modulestore(location).get_item(location)

        self.assertNotIn('html5_sources', own_metadata(module))

    def test_metadata_persistence(self):
        # TODO: create the same test as `test_metadata_not_persistence`,
        # but check persistence for some other module.
        pass


class EntryPageTestCase(TestCase):
    """
    Tests entry pages that aren't specific to a course.
    """
    def setUp(self):
        self.client = AjaxEnabledTestClient()

    def _test_page(self, page, status_code=200):
        resp = self.client.get_html(page)
        self.assertEqual(resp.status_code, status_code)
        _test_no_locations(self, resp, status_code)

    def test_how_it_works(self):
        self._test_page("/howitworks")

    def test_signup(self):
        self._test_page("/signup")

    def test_login(self):
        self._test_page("/signin")

    def test_logout(self):
        # Logout redirects.
        self._test_page("/logout", 302)


def _create_course(test, course_key, course_data):
    """
    Creates a course via an AJAX request and verifies the URL returned in the response.
    """
    course_url = get_url('course_handler', course_key, 'course_key_string')
    response = test.client.ajax_post(course_url, course_data)
    test.assertEqual(response.status_code, 200)
    data = parse_json(response)
    test.assertNotIn('ErrMsg', data)
    test.assertEqual(data['url'], course_url)


def _course_factory_create_course():
    """
    Creates a course via the CourseFactory and returns the locator for it.
    """
    return CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course')


def _get_course_id(course_data):
    """Returns the course ID (org/number/run)."""
    return SlashSeparatedCourseKey(course_data['org'], course_data['number'], course_data['run'])


def _test_no_locations(test, resp, status_code=200, html=True):
    """
    Verifies that "i4x", which appears in old locations, but not
    new locators, does not appear in the HTML response output.
    Used to verify that database refactoring is complete.
    """
    test.assertNotContains(resp, 'i4x', status_code=status_code, html=html)
    if html:
        # For HTML pages, it is nice to call the method with html=True because
        # it checks that the HTML properly parses. However, it won't find i4x usages
        # in JavaScript blocks.
        content = resp.content
        hits = len(re.findall(r"(?<!jump_to/)i4x://", content))
        test.assertEqual(hits, 0, "i4x found outside of LMS jump-to links")

########NEW FILE########
__FILENAME__ = test_core_caching
from cache_toolbox.core import get_cached_content, set_cached_content, del_cached_content
from xmodule.modulestore import Location
from xmodule.contentstore.content import StaticContent
from django.test import TestCase


class Content:
    def __init__(self, location, content):
        self.location = location
        self.content = content

    def get_id(self):
        return self.location.to_deprecated_son()


class CachingTestCase(TestCase):
#   Tests for https://edx.lighthouseapp.com/projects/102637/tickets/112-updating-asset-does-not-refresh-the-cached-copy
    unicodeLocation = Location(u'c4x', u'mitX', u'800', u'run', u'thumbnail', u'monsters.jpg')
    # Note that some of the parts are strings instead of unicode strings
    nonUnicodeLocation = Location('c4x', u'mitX', u'800', u'run', 'thumbnail', 'monsters.jpg')
    mockAsset = Content(unicodeLocation, 'my content')

    def test_put_and_get(self):
        set_cached_content(self.mockAsset)
        self.assertEqual(self.mockAsset.content, get_cached_content(self.unicodeLocation).content,
                         'should be stored in cache with unicodeLocation')
        self.assertEqual(self.mockAsset.content, get_cached_content(self.nonUnicodeLocation).content,
                         'should be stored in cache with nonUnicodeLocation')

    def test_delete(self):
        set_cached_content(self.mockAsset)
        del_cached_content(self.nonUnicodeLocation)
        self.assertEqual(None, get_cached_content(self.unicodeLocation),
                         'should not be stored in cache with unicodeLocation')
        self.assertEqual(None, get_cached_content(self.nonUnicodeLocation),
                         'should not be stored in cache with nonUnicodeLocation')

########NEW FILE########
__FILENAME__ = test_course_listing
"""
Unit tests for getting the list of courses for a user through iterating all courses and
by reversing group name formats.
"""
import random
from chrono import Timer

from django.contrib.auth.models import Group
from django.test import RequestFactory

from contentstore.views.course import _accessible_courses_list, _accessible_courses_list_from_groups
from contentstore.utils import delete_course_and_groups, reverse_course_url
from contentstore.tests.utils import AjaxEnabledTestClient
from student.tests.factories import UserFactory
from student.roles import CourseInstructorRole, CourseStaffRole
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.locations import SlashSeparatedCourseKey

TOTAL_COURSES_COUNT = 500
USER_COURSES_COUNT = 50


class TestCourseListing(ModuleStoreTestCase):
    """
    Unit tests for getting the list of courses for a logged in user
    """
    def setUp(self):
        """
        Add a user and a course
        """
        super(TestCourseListing, self).setUp()
        # create and log in a staff user.
        self.user = UserFactory(is_staff=True)  # pylint: disable=no-member
        self.factory = RequestFactory()
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.user.username, password='test')

    def _create_course_with_access_groups(self, course_location, user=None):
        """
        Create dummy course with 'CourseFactory' and role (instructor/staff) groups
        """
        course = CourseFactory.create(
            org=course_location.org,
            number=course_location.course,
            run=course_location.run
        )

        if user is not None:
            for role in [CourseInstructorRole, CourseStaffRole]:
                role(course.id).add_users(user)

        return course

    def tearDown(self):
        """
        Reverse the setup
        """
        self.client.logout()
        ModuleStoreTestCase.tearDown(self)

    def test_get_course_list(self):
        """
        Test getting courses with new access group format e.g. 'instructor_edx.course.run'
        """
        request = self.factory.get('/course/')
        request.user = self.user

        course_location = SlashSeparatedCourseKey('Org1', 'Course1', 'Run1')
        self._create_course_with_access_groups(course_location, self.user)

        # get courses through iterating all courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # get courses by reversing group name formats
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)
        # check both course lists have same courses
        self.assertEqual(courses_list, courses_list_by_groups)

    def test_get_course_list_with_invalid_course_location(self):
        """
        Test getting courses with invalid course location (course deleted from modulestore).
        """
        request = self.factory.get('/course')
        request.user = self.user

        course_key = SlashSeparatedCourseKey('Org', 'Course', 'Run')
        self._create_course_with_access_groups(course_key, self.user)

        # get courses through iterating all courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # get courses by reversing group name formats
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)
        # check both course lists have same courses
        self.assertEqual(courses_list, courses_list_by_groups)

        # now delete this course and re-add user to instructor group of this course
        delete_course_and_groups(course_key, commit=True)

        CourseInstructorRole(course_key).add_users(self.user)

        # test that get courses through iterating all courses now returns no course
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 0)

        # now test that get courses by reversing group name formats gives 'ItemNotFoundError'
        with self.assertRaises(ItemNotFoundError):
            _accessible_courses_list_from_groups(request)

    def test_course_listing_performance(self):
        """
        Create large number of courses and give access of some of these courses to the user and
        compare the time to fetch accessible courses for the user through traversing all courses and
        reversing django groups
        """
        # create and log in a non-staff user
        self.user = UserFactory()
        request = self.factory.get('/course')
        request.user = self.user
        self.client.login(username=self.user.username, password='test')

        # create list of random course numbers which will be accessible to the user
        user_course_ids = random.sample(range(TOTAL_COURSES_COUNT), USER_COURSES_COUNT)

        # create courses and assign those to the user which have their number in user_course_ids
        for number in range(TOTAL_COURSES_COUNT):
            org = 'Org{0}'.format(number)
            course = 'Course{0}'.format(number)
            run = 'Run{0}'.format(number)
            course_location = SlashSeparatedCourseKey(org, course, run)
            if number in user_course_ids:
                self._create_course_with_access_groups(course_location, self.user)
            else:
                self._create_course_with_access_groups(course_location)

        # time the get courses by iterating through all courses
        with Timer() as iteration_over_courses_time_1:
            courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # time again the get courses by iterating through all courses
        with Timer() as iteration_over_courses_time_2:
            courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # time the get courses by reversing django groups
        with Timer() as iteration_over_groups_time_1:
            courses_list = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # time again the get courses by reversing django groups
        with Timer() as iteration_over_groups_time_2:
            courses_list = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list), USER_COURSES_COUNT)

        # test that the time taken by getting courses through reversing django groups is lower then the time
        # taken by traversing through all courses (if accessible courses are relatively small)
        self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed)
        self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed)

    def test_get_course_list_with_same_course_id(self):
        """
        Test getting courses with same id but with different name case. Then try to delete one of them and
        check that it is properly deleted and other one is accessible
        """
        # create and log in a non-staff user
        self.user = UserFactory()
        request = self.factory.get('/course')
        request.user = self.user
        self.client.login(username=self.user.username, password='test')

        course_location_caps = SlashSeparatedCourseKey('Org', 'COURSE', 'Run')
        self._create_course_with_access_groups(course_location_caps, self.user)

        # get courses through iterating all courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # get courses by reversing group name formats
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)
        # check both course lists have same courses
        self.assertEqual(courses_list, courses_list_by_groups)

        # now create another course with same course_id but different name case
        course_location_camel = SlashSeparatedCourseKey('Org', 'Course', 'Run')
        self._create_course_with_access_groups(course_location_camel, self.user)

        # test that get courses through iterating all courses returns both courses
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 2)

        # test that get courses by reversing group name formats returns both courses
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 2)

        # now delete first course (course_location_caps) and check that it is no longer accessible
        delete_course_and_groups(course_location_caps, commit=True)

        # test that get courses through iterating all courses now returns one course
        courses_list = _accessible_courses_list(request)
        self.assertEqual(len(courses_list), 1)

        # test that get courses by reversing group name formats also returns one course
        courses_list_by_groups = _accessible_courses_list_from_groups(request)
        self.assertEqual(len(courses_list_by_groups), 1)

        # now check that deleted course is not accessible
        outline_url = reverse_course_url('course_handler', course_location_caps)
        response = self.client.get(outline_url, HTTP_ACCEPT='application/json')
        self.assertEqual(response.status_code, 403)

        # now check that other course is accessible
        outline_url = reverse_course_url('course_handler', course_location_camel)
        response = self.client.get(outline_url, HTTP_ACCEPT='application/json')
        self.assertEqual(response.status_code, 200)

########NEW FILE########
__FILENAME__ = test_course_settings
"""
Tests for Studio Course Settings.
"""
import datetime
import json
import copy
import mock

from django.utils.timezone import UTC
from django.test.utils import override_settings

from models.settings.course_details import (CourseDetails, CourseSettingsEncoder)
from models.settings.course_grading import CourseGradingModel
from contentstore.utils import get_modulestore, EXTRA_TAB_PANELS, reverse_course_url, reverse_usage_url
from xmodule.modulestore.tests.factories import CourseFactory

from models.settings.course_metadata import CourseMetadata
from xmodule.fields import Date

from .utils import CourseTestCase
from xmodule.modulestore.django import modulestore
from contentstore.views.component import ADVANCED_COMPONENT_POLICY_KEY


def get_url(course_id, handler_name='settings_handler'):
    return reverse_course_url(handler_name, course_id)

class CourseDetailsTestCase(CourseTestCase):
    """
    Tests the first course settings page (course dates, overview, etc.).
    """
    def test_virgin_fetch(self):
        details = CourseDetails.fetch(self.course.id)
        self.assertEqual(details.org, self.course.location.org, "Org not copied into")
        self.assertEqual(details.course_id, self.course.location.course, "Course_id not copied into")
        self.assertEqual(details.run, self.course.location.name, "Course name not copied into")
        self.assertEqual(details.course_image_name, self.course.course_image)
        self.assertIsNotNone(details.start_date.tzinfo)
        self.assertIsNone(details.end_date, "end date somehow initialized " + str(details.end_date))
        self.assertIsNone(details.enrollment_start, "enrollment_start date somehow initialized " + str(details.enrollment_start))
        self.assertIsNone(details.enrollment_end, "enrollment_end date somehow initialized " + str(details.enrollment_end))
        self.assertIsNone(details.syllabus, "syllabus somehow initialized" + str(details.syllabus))
        self.assertIsNone(details.intro_video, "intro_video somehow initialized" + str(details.intro_video))
        self.assertIsNone(details.effort, "effort somehow initialized" + str(details.effort))

    def test_encoder(self):
        details = CourseDetails.fetch(self.course.id)
        jsondetails = json.dumps(details, cls=CourseSettingsEncoder)
        jsondetails = json.loads(jsondetails)
        self.assertEqual(jsondetails['course_image_name'], self.course.course_image)
        self.assertIsNone(jsondetails['end_date'], "end date somehow initialized ")
        self.assertIsNone(jsondetails['enrollment_start'], "enrollment_start date somehow initialized ")
        self.assertIsNone(jsondetails['enrollment_end'], "enrollment_end date somehow initialized ")
        self.assertIsNone(jsondetails['syllabus'], "syllabus somehow initialized")
        self.assertIsNone(jsondetails['intro_video'], "intro_video somehow initialized")
        self.assertIsNone(jsondetails['effort'], "effort somehow initialized")

    def test_ooc_encoder(self):
        """
        Test the encoder out of its original constrained purpose to see if it functions for general use
        """
        details = {
            'number': 1,
            'string': 'string',
            'datetime': datetime.datetime.now(UTC())
        }
        jsondetails = json.dumps(details, cls=CourseSettingsEncoder)
        jsondetails = json.loads(jsondetails)

        self.assertEquals(1, jsondetails['number'])
        self.assertEqual(jsondetails['string'], 'string')

    def test_update_and_fetch(self):
        jsondetails = CourseDetails.fetch(self.course.id)
        jsondetails.syllabus = "<a href='foo'>bar</a>"
        # encode - decode to convert date fields and other data which changes form
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).syllabus,
            jsondetails.syllabus, "After set syllabus"
        )
        jsondetails.short_description = "Short Description"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).short_description,
            jsondetails.short_description, "After set short_description"
        )
        jsondetails.overview = "Overview"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).overview,
            jsondetails.overview, "After set overview"
        )
        jsondetails.intro_video = "intro_video"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).intro_video,
            jsondetails.intro_video, "After set intro_video"
        )
        jsondetails.effort = "effort"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).effort,
            jsondetails.effort, "After set effort"
        )
        jsondetails.start_date = datetime.datetime(2010, 10, 1, 0, tzinfo=UTC())
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).start_date,
            jsondetails.start_date
        )
        jsondetails.course_image_name = "an_image.jpg"
        self.assertEqual(
            CourseDetails.update_from_json(self.course.id, jsondetails.__dict__, self.user).course_image_name,
            jsondetails.course_image_name
        )

    @override_settings(MKTG_URLS={'ROOT': 'dummy-root'})
    def test_marketing_site_fetch(self):
        settings_details_url = get_url(self.course.id)

        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            response = self.client.get_html(settings_details_url)
            self.assertNotContains(response, "Course Summary Page")
            self.assertNotContains(response, "Send a note to students via email")
            self.assertContains(response, "course summary page will not be viewable")

            self.assertContains(response, "Course Start Date")
            self.assertContains(response, "Course End Date")
            self.assertContains(response, "Enrollment Start Date")
            self.assertContains(response, "Enrollment End Date")
            self.assertContains(response, "not the dates shown on your course summary page")

            self.assertContains(response, "Introducing Your Course")
            self.assertContains(response, "Course Image")
            self.assertContains(response, "Course Short Description")
            self.assertNotContains(response, "Course Overview")
            self.assertNotContains(response, "Course Introduction Video")
            self.assertNotContains(response, "Requirements")

    def test_editable_short_description_fetch(self):
        settings_details_url = get_url(self.course.id)

        with mock.patch.dict('django.conf.settings.FEATURES', {'EDITABLE_SHORT_DESCRIPTION': False}):
            response = self.client.get_html(settings_details_url)
            self.assertNotContains(response, "Course Short Description")


    def test_regular_site_fetch(self):
        settings_details_url = get_url(self.course.id)

        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': False}):
            response = self.client.get_html(settings_details_url)
            self.assertContains(response, "Course Summary Page")
            self.assertContains(response, "Send a note to students via email")
            self.assertNotContains(response, "course summary page will not be viewable")

            self.assertContains(response, "Course Start Date")
            self.assertContains(response, "Course End Date")
            self.assertContains(response, "Enrollment Start Date")
            self.assertContains(response, "Enrollment End Date")
            self.assertNotContains(response, "not the dates shown on your course summary page")

            self.assertContains(response, "Introducing Your Course")
            self.assertContains(response, "Course Image")
            self.assertContains(response, "Course Short Description")
            self.assertContains(response, "Course Overview")
            self.assertContains(response, "Course Introduction Video")
            self.assertContains(response, "Requirements")


class CourseDetailsViewTest(CourseTestCase):
    """
    Tests for modifying content on the first course settings page (course dates, overview, etc.).
    """
    def alter_field(self, url, details, field, val):
        """
        Change the one field to the given value and then invoke the update post to see if it worked.
        """
        setattr(details, field, val)
        # Need to partially serialize payload b/c the mock doesn't handle it correctly
        payload = copy.copy(details.__dict__)
        payload['start_date'] = CourseDetailsViewTest.convert_datetime_to_iso(details.start_date)
        payload['end_date'] = CourseDetailsViewTest.convert_datetime_to_iso(details.end_date)
        payload['enrollment_start'] = CourseDetailsViewTest.convert_datetime_to_iso(details.enrollment_start)
        payload['enrollment_end'] = CourseDetailsViewTest.convert_datetime_to_iso(details.enrollment_end)
        resp = self.client.ajax_post(url, payload)
        self.compare_details_with_encoding(json.loads(resp.content), details.__dict__, field + str(val))

    @staticmethod
    def convert_datetime_to_iso(datetime_obj):
        """
        Use the xblock serializer to convert the datetime
        """
        return Date().to_json(datetime_obj)

    def test_update_and_fetch(self):
        details = CourseDetails.fetch(self.course.id)

        # resp s/b json from here on
        url = get_url(self.course.id)
        resp = self.client.get_json(url)
        self.compare_details_with_encoding(json.loads(resp.content), details.__dict__, "virgin get")

        utc = UTC()
        self.alter_field(url, details, 'start_date', datetime.datetime(2012, 11, 12, 1, 30, tzinfo=utc))
        self.alter_field(url, details, 'start_date', datetime.datetime(2012, 11, 1, 13, 30, tzinfo=utc))
        self.alter_field(url, details, 'end_date', datetime.datetime(2013, 2, 12, 1, 30, tzinfo=utc))
        self.alter_field(url, details, 'enrollment_start', datetime.datetime(2012, 10, 12, 1, 30, tzinfo=utc))

        self.alter_field(url, details, 'enrollment_end', datetime.datetime(2012, 11, 15, 1, 30, tzinfo=utc))
        self.alter_field(url, details, 'short_description', "Short Description")
        self.alter_field(url, details, 'overview', "Overview")
        self.alter_field(url, details, 'intro_video', "intro_video")
        self.alter_field(url, details, 'effort', "effort")
        self.alter_field(url, details, 'course_image_name', "course_image_name")

    def compare_details_with_encoding(self, encoded, details, context):
        """
        compare all of the fields of the before and after dicts
        """
        self.compare_date_fields(details, encoded, context, 'start_date')
        self.compare_date_fields(details, encoded, context, 'end_date')
        self.compare_date_fields(details, encoded, context, 'enrollment_start')
        self.compare_date_fields(details, encoded, context, 'enrollment_end')
        self.assertEqual(details['short_description'], encoded['short_description'], context + " short_description not ==")
        self.assertEqual(details['overview'], encoded['overview'], context + " overviews not ==")
        self.assertEqual(details['intro_video'], encoded.get('intro_video', None), context + " intro_video not ==")
        self.assertEqual(details['effort'], encoded['effort'], context + " efforts not ==")
        self.assertEqual(details['course_image_name'], encoded['course_image_name'], context + " images not ==")

    def compare_date_fields(self, details, encoded, context, field):
        """
        Compare the given date fields between the before and after doing json deserialization
        """
        if details[field] is not None:
            date = Date()
            if field in encoded and encoded[field] is not None:
                dt1 = date.from_json(encoded[field])
                dt2 = details[field]

                self.assertEqual(dt1, dt2, msg="{} != {} at {}".format(dt1, dt2, context))
            else:
                self.fail(field + " missing from encoded but in details at " + context)
        elif field in encoded and encoded[field] is not None:
            self.fail(field + " included in encoding but missing from details at " + context)


class CourseGradingTest(CourseTestCase):
    """
    Tests for the course settings grading page.
    """
    def test_initial_grader(self):
        test_grader = CourseGradingModel(self.course)
        self.assertIsNotNone(test_grader.graders)
        self.assertIsNotNone(test_grader.grade_cutoffs)

    def test_fetch_grader(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        self.assertIsNotNone(test_grader.graders, "No graders")
        self.assertIsNotNone(test_grader.grade_cutoffs, "No cutoffs")

        for i, grader in enumerate(test_grader.graders):
            subgrader = CourseGradingModel.fetch_grader(self.course.id, i)
            self.assertDictEqual(grader, subgrader, str(i) + "th graders not equal")

    def test_update_from_json(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "Noop update")

        test_grader.graders[0]['weight'] = test_grader.graders[0].get('weight') * 2
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "Weight[0] * 2")

        test_grader.grade_cutoffs['D'] = 0.3
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "cutoff add D")

        test_grader.grace_period = {'hours': 4, 'minutes': 5, 'seconds': 0}
        altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user)
        self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "4 hour grace period")

    def test_update_grader_from_json(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        altered_grader = CourseGradingModel.update_grader_from_json(
            self.course.id, test_grader.graders[1], self.user
        )
        self.assertDictEqual(test_grader.graders[1], altered_grader, "Noop update")

        test_grader.graders[1]['min_count'] = test_grader.graders[1].get('min_count') + 2
        altered_grader = CourseGradingModel.update_grader_from_json(
            self.course.id, test_grader.graders[1], self.user)
        self.assertDictEqual(test_grader.graders[1], altered_grader, "min_count[1] + 2")

        test_grader.graders[1]['drop_count'] = test_grader.graders[1].get('drop_count') + 1
        altered_grader = CourseGradingModel.update_grader_from_json(
            self.course.id, test_grader.graders[1], self.user)
        self.assertDictEqual(test_grader.graders[1], altered_grader, "drop_count[1] + 2")

    def test_update_cutoffs_from_json(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user)
        # Unlike other tests, need to actually perform a db fetch for this test since update_cutoffs_from_json
        #  simply returns the cutoffs you send into it, rather than returning the db contents.
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "Noop update")

        test_grader.grade_cutoffs['D'] = 0.3
        CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user)
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "cutoff add D")

        test_grader.grade_cutoffs['Pass'] = 0.75
        CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user)
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "cutoff change 'Pass'")

    def test_delete_grace_period(self):
        test_grader = CourseGradingModel.fetch(self.course.id)
        CourseGradingModel.update_grace_period_from_json(
            self.course.id, test_grader.grace_period, self.user
        )
        # update_grace_period_from_json doesn't return anything, so query the db for its contents.
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertEqual(test_grader.grace_period, altered_grader.grace_period, "Noop update")

        test_grader.grace_period = {'hours': 15, 'minutes': 5, 'seconds': 30}
        CourseGradingModel.update_grace_period_from_json(
            self.course.id, test_grader.grace_period, self.user)
        altered_grader = CourseGradingModel.fetch(self.course.id)
        self.assertDictEqual(test_grader.grace_period, altered_grader.grace_period, "Adding in a grace period")

        test_grader.grace_period = {'hours': 1, 'minutes': 10, 'seconds': 0}
        # Now delete the grace period
        CourseGradingModel.delete_grace_period(self.course.id, self.user)
        # update_grace_period_from_json doesn't return anything, so query the db for its contents.
        altered_grader = CourseGradingModel.fetch(self.course.id)
        # Once deleted, the grace period should simply be None
        self.assertEqual(None, altered_grader.grace_period, "Delete grace period")

    def test_update_section_grader_type(self):
        # Get the descriptor and the section_grader_type and assert they are the default values
        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
        section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)

        self.assertEqual('notgraded', section_grader_type['graderType'])
        self.assertEqual(None, descriptor.format)
        self.assertEqual(False, descriptor.graded)

        # Change the default grader type to Homework, which should also mark the section as graded
        CourseGradingModel.update_section_grader_type(self.course, 'Homework', self.user)
        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
        section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)

        self.assertEqual('Homework', section_grader_type['graderType'])
        self.assertEqual('Homework', descriptor.format)
        self.assertEqual(True, descriptor.graded)

        # Change the grader type back to notgraded, which should also unmark the section as graded
        CourseGradingModel.update_section_grader_type(self.course, 'notgraded', self.user)
        descriptor = get_modulestore(self.course.location).get_item(self.course.location)
        section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location)

        self.assertEqual('notgraded', section_grader_type['graderType'])
        self.assertEqual(None, descriptor.format)
        self.assertEqual(False, descriptor.graded)

    def test_get_set_grader_types_ajax(self):
        """
        Test configuring the graders via ajax calls
        """
        grader_type_url_base = get_url(self.course.id, 'grading_handler')
        # test get whole
        response = self.client.get_json(grader_type_url_base)
        whole_model = json.loads(response.content)
        self.assertIn('graders', whole_model)
        self.assertIn('grade_cutoffs', whole_model)
        self.assertIn('grace_period', whole_model)
        # test post/update whole
        whole_model['grace_period'] = {'hours': 1, 'minutes': 30, 'seconds': 0}
        response = self.client.ajax_post(grader_type_url_base, whole_model)
        self.assertEqual(200, response.status_code)
        response = self.client.get_json(grader_type_url_base)
        whole_model = json.loads(response.content)
        self.assertEqual(whole_model['grace_period'], {'hours': 1, 'minutes': 30, 'seconds': 0})
        # test get one grader
        self.assertGreater(len(whole_model['graders']), 1)  # ensure test will make sense
        response = self.client.get_json(grader_type_url_base + '/1')
        grader_sample = json.loads(response.content)
        self.assertEqual(grader_sample, whole_model['graders'][1])
        # test add grader
        new_grader = {
            "type": "Extra Credit",
            "min_count": 1,
            "drop_count": 2,
            "short_label": None,
            "weight": 15,
        }
        response = self.client.ajax_post(
            '{}/{}'.format(grader_type_url_base, len(whole_model['graders'])),
            new_grader
        )
        self.assertEqual(200, response.status_code)
        grader_sample = json.loads(response.content)
        new_grader['id'] = len(whole_model['graders'])
        self.assertEqual(new_grader, grader_sample)
        # test delete grader
        response = self.client.delete(grader_type_url_base + '/1', HTTP_ACCEPT="application/json")
        self.assertEqual(204, response.status_code)
        response = self.client.get_json(grader_type_url_base)
        updated_model = json.loads(response.content)
        new_grader['id'] -= 1  # one fewer and the id mutates
        self.assertIn(new_grader, updated_model['graders'])
        self.assertNotIn(whole_model['graders'][1], updated_model['graders'])

    def setup_test_set_get_section_grader_ajax(self):
        """
        Populate the course, grab a section, get the url for the assignment type access
        """
        self.populate_course()
        sequential_usage_key = self.course.id.make_usage_key("sequential", None)
        sections = get_modulestore(self.course.id).get_items(sequential_usage_key)
        # see if test makes sense
        self.assertGreater(len(sections), 0, "No sections found")
        section = sections[0]  # just take the first one
        return reverse_usage_url('xblock_handler', section.location)

    def test_set_get_section_grader_ajax(self):
        """
        Test setting and getting section grades via the grade as url
        """
        grade_type_url = self.setup_test_set_get_section_grader_ajax()
        response = self.client.ajax_post(grade_type_url, {'graderType': u'Homework'})
        self.assertEqual(200, response.status_code)
        response = self.client.get_json(grade_type_url + '?fields=graderType')
        self.assertEqual(json.loads(response.content).get('graderType'), u'Homework')
        # and unset
        response = self.client.ajax_post(grade_type_url, {'graderType': u'notgraded'})
        self.assertEqual(200, response.status_code)
        response = self.client.get_json(grade_type_url + '?fields=graderType')
        self.assertEqual(json.loads(response.content).get('graderType'), u'notgraded')


class CourseMetadataEditingTest(CourseTestCase):
    """
    Tests for CourseMetadata.
    """
    def setUp(self):
        CourseTestCase.setUp(self)
        self.fullcourse = CourseFactory.create(org='edX', course='999', display_name='Robot Super Course')
        self.course_setting_url = get_url(self.course.id, 'advanced_settings_handler')
        self.fullcourse_setting_url = get_url(self.fullcourse.id, 'advanced_settings_handler')

    def test_fetch_initial_fields(self):
        test_model = CourseMetadata.fetch(self.course)
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")

        test_model = CourseMetadata.fetch(self.fullcourse)
        self.assertNotIn('graceperiod', test_model, 'blacklisted field leaked in')
        self.assertIn('display_name', test_model, 'full missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('rerandomize', test_model, 'Missing rerandomize metadata field')
        self.assertIn('showanswer', test_model, 'showanswer field ')
        self.assertIn('xqa_key', test_model, 'xqa_key field ')

    def test_update_from_json(self):
        test_model = CourseMetadata.update_from_json(
            self.course,
            {
                "advertised_start": "start A",
                "days_early_for_beta": 2,
            },
            user=self.user
         )
        self.update_check(test_model)
        # try fresh fetch to ensure persistence
        fresh = modulestore('direct').get_course(self.course.id)
        test_model = CourseMetadata.fetch(fresh)
        self.update_check(test_model)
        # now change some of the existing metadata
        test_model = CourseMetadata.update_from_json(
            fresh,
            {
                "advertised_start": "start B",
                "display_name": "jolly roger",
            },
            user=self.user
        )
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'jolly roger', "not expected value")
        self.assertIn('advertised_start', test_model, 'Missing revised advertised_start metadata field')
        self.assertEqual(test_model['advertised_start'], 'start B', "advertised_start not expected value")

    def update_check(self, test_model):
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('advertised_start', test_model, 'Missing new advertised_start metadata field')
        self.assertEqual(test_model['advertised_start'], 'start A', "advertised_start not expected value")
        self.assertIn('days_early_for_beta', test_model, 'Missing days_early_for_beta metadata field')
        self.assertEqual(test_model['days_early_for_beta'], 2, "days_early_for_beta not expected value")

    def test_delete_key(self):
        test_model = CourseMetadata.update_from_json(
            self.fullcourse,
            {"unsetKeys": ['showanswer', 'xqa_key']},
            user=self.user
        )
        # ensure no harm
        self.assertNotIn('graceperiod', test_model, 'blacklisted field leaked in')
        self.assertIn('display_name', test_model, 'full missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('rerandomize', test_model, 'Missing rerandomize metadata field')
        # check for deletion effectiveness
        self.assertEqual('finished', test_model['showanswer'], 'showanswer field still in')
        self.assertEqual(None, test_model['xqa_key'], 'xqa_key field still in')

    def test_http_fetch_initial_fields(self):
        response = self.client.get_json(self.course_setting_url)
        test_model = json.loads(response.content)
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")

        response = self.client.get_json(self.fullcourse_setting_url)
        test_model = json.loads(response.content)
        self.assertNotIn('graceperiod', test_model, 'blacklisted field leaked in')
        self.assertIn('display_name', test_model, 'full missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'Robot Super Course', "not expected value")
        self.assertIn('rerandomize', test_model, 'Missing rerandomize metadata field')
        self.assertIn('showanswer', test_model, 'showanswer field ')
        self.assertIn('xqa_key', test_model, 'xqa_key field ')

    def test_http_update_from_json(self):
        response = self.client.ajax_post(self.course_setting_url, {
            "advertised_start": "start A",
            "testcenter_info": {"c": "test"},
            "days_early_for_beta": 2,
            "unsetKeys": ['showanswer', 'xqa_key'],
        })
        test_model = json.loads(response.content)
        self.update_check(test_model)
        self.assertEqual('finished', test_model['showanswer'], 'showanswer field still in')
        self.assertEqual(None, test_model['xqa_key'], 'xqa_key field still in')

        response = self.client.get_json(self.course_setting_url)
        test_model = json.loads(response.content)
        self.update_check(test_model)
        # now change some of the existing metadata
        response = self.client.ajax_post(self.course_setting_url, {
            "advertised_start": "start B",
            "display_name": "jolly roger"
        })
        test_model = json.loads(response.content)
        self.assertIn('display_name', test_model, 'Missing editable metadata field')
        self.assertEqual(test_model['display_name'], 'jolly roger', "not expected value")
        self.assertIn('advertised_start', test_model, 'Missing revised advertised_start metadata field')
        self.assertEqual(test_model['advertised_start'], 'start B', "advertised_start not expected value")

    def test_advanced_components_munge_tabs(self):
        """
        Test that adding and removing specific advanced components adds and removes tabs.
        """
        self.assertNotIn(EXTRA_TAB_PANELS.get("open_ended"), self.course.tabs)
        self.assertNotIn(EXTRA_TAB_PANELS.get("notes"), self.course.tabs)
        self.client.ajax_post(self.course_setting_url, {
            ADVANCED_COMPONENT_POLICY_KEY: ["combinedopenended"]
        })
        course = modulestore().get_course(self.course.id)
        self.assertIn(EXTRA_TAB_PANELS.get("open_ended"), course.tabs)
        self.assertNotIn(EXTRA_TAB_PANELS.get("notes"), course.tabs)
        self.client.ajax_post(self.course_setting_url, {
            ADVANCED_COMPONENT_POLICY_KEY: []
        })
        course = modulestore().get_course(self.course.id)
        self.assertNotIn(EXTRA_TAB_PANELS.get("open_ended"), course.tabs)


class CourseGraderUpdatesTest(CourseTestCase):
    """
    Test getting, deleting, adding, & updating graders
    """
    def setUp(self):
        """Compute the url to use in tests"""
        super(CourseGraderUpdatesTest, self).setUp()
        self.url = get_url(self.course.id, 'grading_handler')
        self.starting_graders = CourseGradingModel(self.course).graders

    def test_get(self):
        """Test getting a specific grading type record."""
        resp = self.client.get_json(self.url + '/0')
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(self.starting_graders[0], obj)

    def test_delete(self):
        """Test deleting a specific grading type record."""
        resp = self.client.delete(self.url + '/0', HTTP_ACCEPT="application/json")
        self.assertEqual(resp.status_code, 204)
        current_graders = CourseGradingModel.fetch(self.course.id).graders
        self.assertNotIn(self.starting_graders[0], current_graders)
        self.assertEqual(len(self.starting_graders) - 1, len(current_graders))

    def test_update(self):
        """Test updating a specific grading type record."""
        grader = {
            "id": 0,
            "type": "manual",
            "min_count": 5,
            "drop_count": 10,
            "short_label": "yo momma",
            "weight": 17.3,
        }
        resp = self.client.ajax_post(self.url + '/0', grader)
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(obj, grader)
        current_graders = CourseGradingModel.fetch(self.course.id).graders
        self.assertEqual(len(self.starting_graders), len(current_graders))

    def test_add(self):
        """Test adding a grading type record."""
        # the same url works for changing the whole grading model (graceperiod, cutoffs, and grading types) when
        # the grading_index is None; thus, using None to imply adding a grading_type doesn't work; so, it uses an
        # index out of bounds to imply create item.
        grader = {
            "type": "manual",
            "min_count": 5,
            "drop_count": 10,
            "short_label": "yo momma",
            "weight": 17.3,
        }
        resp = self.client.ajax_post('{}/{}'.format(self.url, len(self.starting_graders) + 1), grader)
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(obj['id'], len(self.starting_graders))
        del obj['id']
        self.assertEqual(obj, grader)
        current_graders = CourseGradingModel.fetch(self.course.id).graders
        self.assertEqual(len(self.starting_graders) + 1, len(current_graders))

########NEW FILE########
__FILENAME__ = test_crud
import unittest

from xmodule import templates
from xmodule.modulestore.tests import persistent_factories
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.django import modulestore, loc_mapper, clear_existing_modulestores
from xmodule.seq_module import SequenceDescriptor
from xmodule.capa_module import CapaDescriptor
from xmodule.modulestore.locator import BlockUsageLocator, LocalId
from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateCourseError
from xmodule.html_module import HtmlDescriptor
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase


class TemplateTests(unittest.TestCase):
    """
    Test finding and using the templates (boilerplates) for xblocks.
    """

    def setUp(self):
        clear_existing_modulestores()  # redundant w/ cleanup but someone was getting errors
        self.addCleanup(ModuleStoreTestCase.drop_mongo_collections, 'split')
        self.addCleanup(clear_existing_modulestores)

    def test_get_templates(self):
        found = templates.all_templates()
        self.assertIsNotNone(found.get('course'))
        self.assertIsNotNone(found.get('about'))
        self.assertIsNotNone(found.get('html'))
        self.assertIsNotNone(found.get('problem'))
        self.assertEqual(len(found.get('course')), 0)
        self.assertEqual(len(found.get('about')), 1)
        self.assertGreaterEqual(len(found.get('html')), 2)
        self.assertGreaterEqual(len(found.get('problem')), 10)
        dropdown = None
        for template in found['problem']:
            self.assertIn('metadata', template)
            self.assertIn('display_name', template['metadata'])
            if template['metadata']['display_name'] == 'Dropdown':
                dropdown = template
                break
        self.assertIsNotNone(dropdown)
        self.assertIn('markdown', dropdown['metadata'])
        self.assertIn('data', dropdown)
        self.assertRegexpMatches(dropdown['metadata']['markdown'], r'^Dropdown.*')
        self.assertRegexpMatches(dropdown['data'], r'<problem>\s*<p>Dropdown.*')

    def test_get_some_templates(self):
        self.assertEqual(len(SequenceDescriptor.templates()), 0)
        self.assertGreater(len(HtmlDescriptor.templates()), 0)
        self.assertIsNone(SequenceDescriptor.get_template('doesntexist.yaml'))
        self.assertIsNone(HtmlDescriptor.get_template('doesntexist.yaml'))
        self.assertIsNotNone(HtmlDescriptor.get_template('announcement.yaml'))

    def test_factories(self):
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='tempcourse', org='testx',
            display_name='fun test course', user_id='testbot'
        )
        self.assertIsInstance(test_course, CourseDescriptor)
        self.assertEqual(test_course.display_name, 'fun test course')
        index_info = modulestore('split').get_course_index_info(test_course.id)
        self.assertEqual(index_info['org'], 'testx')
        self.assertEqual(index_info['offering'], 'tempcourse')

        test_chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
            parent_location=test_course.location)
        self.assertIsInstance(test_chapter, SequenceDescriptor)
        # refetch parent which should now point to child
        test_course = modulestore('split').get_course(test_course.id.version_agnostic())
        self.assertIn(test_chapter.location, test_course.children)

        with self.assertRaises(DuplicateCourseError):
            persistent_factories.PersistentCourseFactory.create(
                offering='tempcourse', org='testx',
                display_name='fun test course', user_id='testbot'
            )

    def test_temporary_xblocks(self):
        """
        Test create_xblock to create non persisted xblocks
        """
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='tempcourse', org='testx',
            display_name='fun test course', user_id='testbot'
        )

        test_chapter = modulestore('split').create_xblock(
            test_course.system, 'chapter', {'display_name': 'chapter n'}, parent_xblock=test_course
        )
        self.assertIsInstance(test_chapter, SequenceDescriptor)
        self.assertEqual(test_chapter.display_name, 'chapter n')
        self.assertIn(test_chapter, test_course.get_children())

        # test w/ a definition (e.g., a problem)
        test_def_content = '<problem>boo</problem>'
        test_problem = modulestore('split').create_xblock(
            test_course.system, 'problem', {'data': test_def_content}, parent_xblock=test_chapter
        )
        self.assertIsInstance(test_problem, CapaDescriptor)
        self.assertEqual(test_problem.data, test_def_content)
        self.assertIn(test_problem, test_chapter.get_children())
        test_problem.display_name = 'test problem'
        self.assertEqual(test_problem.display_name, 'test problem')

    def test_persist_dag(self):
        """
        try saving temporary xblocks
        """
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='tempcourse', org='testx',
            display_name='fun test course', user_id='testbot'
        )
        test_chapter = modulestore('split').create_xblock(
            test_course.system, 'chapter', {'display_name': 'chapter n'}, parent_xblock=test_course
        )
        self.assertEqual(test_chapter.display_name, 'chapter n')
        test_def_content = '<problem>boo</problem>'
        # create child
        new_block = modulestore('split').create_xblock(
            test_course.system,
            'problem',
            fields={
                'data': test_def_content,
                'display_name': 'problem'
            },
            parent_xblock=test_chapter
        )
        self.assertIsNotNone(new_block.definition_locator)
        self.assertTrue(isinstance(new_block.definition_locator.definition_id, LocalId))
        # better to pass in persisted parent over the subdag so
        # subdag gets the parent pointer (otherwise 2 ops, persist dag, update parent children,
        # persist parent
        persisted_course = modulestore('split').persist_xblock_dag(test_course, 'testbot')
        self.assertEqual(len(persisted_course.children), 1)
        persisted_chapter = persisted_course.get_children()[0]
        self.assertEqual(persisted_chapter.category, 'chapter')
        self.assertEqual(persisted_chapter.display_name, 'chapter n')
        self.assertEqual(len(persisted_chapter.children), 1)
        persisted_problem = persisted_chapter.get_children()[0]
        self.assertEqual(persisted_problem.category, 'problem')
        self.assertEqual(persisted_problem.data, test_def_content)
        # update it
        persisted_problem.display_name = 'altered problem'
        persisted_problem = modulestore('split').persist_xblock_dag(persisted_problem, 'testbot')
        self.assertEqual(persisted_problem.display_name, 'altered problem')

    def test_delete_course(self):
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='history.doomed', org='edu.harvard',
            display_name='doomed test course',
            user_id='testbot')
        persistent_factories.ItemFactory.create(display_name='chapter 1',
            parent_location=test_course.location)

        id_locator = test_course.id.for_branch('draft')
        guid_locator = test_course.location.course_agnostic()
        # verify it can be retrieved by id
        self.assertIsInstance(modulestore('split').get_course(id_locator), CourseDescriptor)
        # and by guid
        self.assertIsInstance(modulestore('split').get_item(guid_locator), CourseDescriptor)
        modulestore('split').delete_course(id_locator)
        # test can no longer retrieve by id
        self.assertRaises(ItemNotFoundError, modulestore('split').get_course, id_locator)
        # but can by guid
        self.assertIsInstance(modulestore('split').get_item(guid_locator), CourseDescriptor)

    def test_block_generations(self):
        """
        Test get_block_generations
        """
        test_course = persistent_factories.PersistentCourseFactory.create(
            offering='history.hist101', org='edu.harvard',
            display_name='history test course',
            user_id='testbot'
        )
        chapter = persistent_factories.ItemFactory.create(display_name='chapter 1',
            parent_location=test_course.location, user_id='testbot')
        sub = persistent_factories.ItemFactory.create(display_name='subsection 1',
            parent_location=chapter.location, user_id='testbot', category='vertical')
        first_problem = persistent_factories.ItemFactory.create(
            display_name='problem 1', parent_location=sub.location, user_id='testbot', category='problem',
            data="<problem></problem>"
        )
        first_problem.max_attempts = 3
        first_problem.save()  # decache the above into the kvs
        updated_problem = modulestore('split').update_item(first_problem, '**replace_user**')
        self.assertIsNotNone(updated_problem.previous_version)
        self.assertEqual(updated_problem.previous_version, first_problem.update_version)
        self.assertNotEqual(updated_problem.update_version, first_problem.update_version)
        updated_loc = modulestore('split').delete_item(updated_problem.location, 'testbot', delete_children=True)

        second_problem = persistent_factories.ItemFactory.create(
            display_name='problem 2',
            parent_location=BlockUsageLocator.make_relative(
                updated_loc, block_type='problem', block_id=sub.location.block_id
            ),
            user_id='testbot', category='problem',
            data="<problem></problem>"
        )

        # course root only updated 2x
        version_history = modulestore('split').get_block_generations(test_course.location)
        self.assertEqual(version_history.locator.version_guid, test_course.location.version_guid)
        self.assertEqual(len(version_history.children), 1)
        self.assertEqual(version_history.children[0].children, [])
        self.assertEqual(version_history.children[0].locator.version_guid, chapter.location.version_guid)

        # sub changed on add, add problem, delete problem, add problem in strict linear seq
        version_history = modulestore('split').get_block_generations(sub.location)
        self.assertEqual(len(version_history.children), 1)
        self.assertEqual(len(version_history.children[0].children), 1)
        self.assertEqual(len(version_history.children[0].children[0].children), 1)
        self.assertEqual(len(version_history.children[0].children[0].children[0].children), 0)

        # first and second problem may show as same usage_id; so, need to ensure their histories are right
        version_history = modulestore('split').get_block_generations(updated_problem.location)
        self.assertEqual(version_history.locator.version_guid, first_problem.location.version_guid)
        self.assertEqual(len(version_history.children), 1)  # updated max_attempts
        self.assertEqual(len(version_history.children[0].children), 0)

        version_history = modulestore('split').get_block_generations(second_problem.location)
        self.assertNotEqual(version_history.locator.version_guid, first_problem.location.version_guid)

    def test_split_inject_loc_mapper(self):
        """
        Test that creating a loc_mapper causes it to automatically attach to the split mongo store
        """
        # instantiate location mapper before split
        mapper = loc_mapper()
        # split must inject the location mapper itself since the mapper existed before it did
        self.assertEqual(modulestore('split').loc_mapper, mapper)

    def test_loc_inject_into_split(self):
        """
        Test that creating a loc_mapper causes it to automatically attach to the split mongo store
        """
        # force instantiation of split modulestore before there's a location mapper and verify
        # it has no pointer to loc mapper
        self.assertIsNone(modulestore('split').loc_mapper)
        # force instantiation of location mapper which must inject itself into the split
        mapper = loc_mapper()
        self.assertEqual(modulestore('split').loc_mapper, mapper)


########NEW FILE########
__FILENAME__ = test_export_git
"""
Test the ability to export courses to xml from studio
"""

import copy
import os
import shutil
import subprocess
from uuid import uuid4

from django.conf import settings
from django.test.utils import override_settings
from pymongo import MongoClient

from .utils import CourseTestCase
import contentstore.git_export_utils as git_export_utils
from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestExportGit(CourseTestCase):
    """
    Tests pushing a course to a git repository
    """

    def setUp(self):
        """
        Setup test course, user, and url.
        """
        super(TestExportGit, self).setUp()
        self.course_module = modulestore().get_course(self.course.id)
        self.test_url = reverse_course_url('export_git', self.course.id)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_giturl_missing(self):
        """
        Test to make sure an appropriate error is displayed
        if course hasn't set giturl.
        """
        response = self.client.get(self.test_url)
        self.assertEqual(200, response.status_code)
        self.assertIn(
            ('giturl must be defined in your '
             'course settings before you can export to git.'),
            response.content
        )

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertEqual(200, response.status_code)
        self.assertIn(
            ('giturl must be defined in your '
             'course settings before you can export to git.'),
            response.content
        )

    def test_course_export_failures(self):
        """
        Test failed course export response.
        """
        self.course_module.giturl = 'foobar'
        get_modulestore(self.course_module.location).update_item(self.course_module)

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertIn('Export Failed:', response.content)

    def test_exception_translation(self):
        """
        Regression test for making sure errors are properly stringified
        """
        self.course_module.giturl = 'foobar'
        get_modulestore(self.course_module.location).update_item(self.course_module)

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertNotIn('django.utils.functional.__proxy__', response.content)

    def test_course_export_success(self):
        """
        Test successful course export response.
        """
        # Build out local bare repo, and set course git url to it
        repo_dir = os.path.abspath(git_export_utils.GIT_REPO_EXPORT_DIR)
        os.mkdir(repo_dir)
        self.addCleanup(shutil.rmtree, repo_dir)

        bare_repo_dir = '{0}/test_repo.git'.format(
            os.path.abspath(git_export_utils.GIT_REPO_EXPORT_DIR))
        os.mkdir(bare_repo_dir)
        self.addCleanup(shutil.rmtree, bare_repo_dir)

        subprocess.check_output(['git', '--bare', 'init', ], cwd=bare_repo_dir)

        self.populate_course()
        self.course_module.giturl = 'file://{}'.format(bare_repo_dir)
        get_modulestore(self.course_module.location).update_item(self.course_module)

        response = self.client.get('{}?action=push'.format(self.test_url))
        self.assertIn('Export Succeeded', response.content)

########NEW FILE########
__FILENAME__ = test_i18n
from unittest import skip

from django.contrib.auth.models import User
from django.test.utils import override_settings

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient


@override_settings(MODULESTORE=TEST_MODULESTORE)
class InternationalizationTest(ModuleStoreTestCase):
    """
    Tests to validate Internationalization.
    """

    def setUp(self):
        """
        These tests need a user in the DB so that the django Test Client
        can log them in.
        They inherit from the ModuleStoreTestCase class so that the mongodb collection
        will be cleared out before each test case execution and deleted
        afterwards.
        """
        self.uname = 'testuser'
        self.email = 'test+courses@edx.org'
        self.password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(self.uname, self.email, self.password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.course_data = {
            'org': 'MITx',
            'number': '999',
            'display_name': 'Robot Super Course',
        }

    def test_course_plain_english(self):
        """Test viewing the index page with no courses"""
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.uname, password=self.password)

        resp = self.client.get_html('/course/')
        self.assertContains(resp,
                            '<h1 class="page-header">My Courses</h1>',
                            status_code=200,
                            html=True)

    def test_course_explicit_english(self):
        """Test viewing the index page with no courses"""
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.uname, password=self.password)

        resp = self.client.get_html('/course/',
                               {},
                               HTTP_ACCEPT_LANGUAGE='en'
                               )

        self.assertContains(resp,
                            '<h1 class="page-header">My Courses</h1>',
                            status_code=200,
                            html=True)

    # ****
    # NOTE:
    # ****
    #
    # This test will break when we replace this fake 'test' language
    # with actual Esperanto. This test will need to be updated with
    # actual Esperanto at that time.
    # Test temporarily disable since it depends on creation of dummy strings
    @skip
    def test_course_with_accents(self):
        """Test viewing the index page with no courses"""
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.uname, password=self.password)

        resp = self.client.get_html(
            '/course/',
            {},
            HTTP_ACCEPT_LANGUAGE='eo'
        )

        TEST_STRING = (
            u'<h1 class="title-1">'
            u'My \xc7\xf6\xfcrs\xe9s L#'
            u'</h1>'
        )

        self.assertContains(resp,
                            TEST_STRING,
                            status_code=200,
                            html=True)

########NEW FILE########
__FILENAME__ = test_import
# -*- coding: utf-8 -*-
# pylint: disable=E1101
"""
Tests for import_from_xml using the mongo modulestore.
"""

from django.test.client import Client
from django.test.utils import override_settings
from django.conf import settings
from path import path
import copy

from django.contrib.auth.models import User

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE

from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import _CONTENTSTORE

from xmodule.exceptions import NotFoundError
from uuid import uuid4
from pymongo import MongoClient

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreImportTest(ModuleStoreTestCase):
    """
    Tests that rely on the toy and test_import_course courses.
    NOTE: refactor using CourseFactory so they do not.
    """
    def setUp(self):

        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True

        # Save the data that we've just changed to the db.
        self.user.save()

        self.client = Client()
        self.client.login(username=uname, password=password)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def load_test_import_course(self):
        '''
        Load the standard course used to test imports
        (for do_import_static=False behavior).
        '''
        content_store = contentstore()
        module_store = modulestore('direct')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['test_import_course'],
            static_content_store=content_store,
            do_import_static=False,
            verbose=True,
        )
        course_id = SlashSeparatedCourseKey('edX', 'test_import_course', '2012_Fall')
        course = module_store.get_course(course_id)
        self.assertIsNotNone(course)

        return module_store, content_store, course

    def test_import_course_into_similar_namespace(self):
        # Checks to make sure that a course with an org/course like
        # edx/course can be imported into a namespace with an org/course
        # like edx/course_name
        module_store, __, course = self.load_test_import_course()
        __, course_items = import_from_xml(
            module_store,
            'common/test/data',
            ['test_import_course_2'],
            target_course_id=course.id,
            verbose=True,
        )
        self.assertEqual(len(course_items), 1)

    def test_unicode_chars_in_course_name_import(self):
        """
        # Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
        """
        module_store = modulestore('direct')
        course_id = SlashSeparatedCourseKey(u'Юникода', u'unicode_course', u'échantillon')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['2014_Uni'],
            target_course_id=course_id
        )

        course = module_store.get_course(course_id)
        self.assertIsNotNone(course)

        # test that course 'display_name' same as imported course 'display_name'
        self.assertEqual(course.display_name, u"Φυσικά το όνομα Unicode")

    def test_static_import(self):
        '''
        Stuff in static_import should always be imported into contentstore
        '''
        _, content_store, course = self.load_test_import_course()

        # make sure we have ONE asset in our contentstore ("should_be_imported.html")
        all_assets, count = content_store.get_all_content_for_course(course.id)
        print "len(all_assets)=%d" % len(all_assets)
        self.assertEqual(len(all_assets), 1)
        self.assertEqual(count, 1)

        content = None
        try:
            location = AssetLocation.from_deprecated_string(
                '/c4x/edX/test_import_course/asset/should_be_imported.html'
            )
            content = content_store.find(location)
        except NotFoundError:
            pass

        self.assertIsNotNone(content)

        # make sure course.static_asset_path is correct
        print "static_asset_path = {0}".format(course.static_asset_path)
        self.assertEqual(course.static_asset_path, 'test_import_course')

    def test_asset_import_nostatic(self):
        '''
        This test validates that an image asset is NOT imported when do_import_static=False
        '''
        content_store = contentstore()

        module_store = modulestore('direct')
        import_from_xml(module_store, 'common/test/data/', ['toy'], static_content_store=content_store, do_import_static=False, verbose=True)

        course = module_store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))

        # make sure we have NO assets in our contentstore
        all_assets, count = content_store.get_all_content_for_course(course.id)
        self.assertEqual(len(all_assets), 0)
        self.assertEqual(count, 0)

    def test_no_static_link_rewrites_on_import(self):
        module_store = modulestore('direct')
        _, courses = import_from_xml(module_store, 'common/test/data/', ['toy'], do_import_static=False, verbose=True)
        course_key = courses[0].id

        handouts = module_store.get_item(course_key.make_usage_key('course_info', 'handouts'))
        self.assertIn('/static/', handouts.data)

        handouts = module_store.get_item(course_key.make_usage_key('html', 'toyhtml'))
        self.assertIn('/static/', handouts.data)

    def test_tab_name_imports_correctly(self):
        _module_store, _content_store, course = self.load_test_import_course()
        print "course tabs = {0}".format(course.tabs)
        self.assertEqual(course.tabs[2]['name'], 'Syllabus')

    def test_rewrite_reference_list(self):
        module_store = modulestore('direct')
        target_course_id = SlashSeparatedCourseKey('testX', 'conditional_copy', 'copy_run')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['conditional'],
            target_course_id=target_course_id
        )
        conditional_module = module_store.get_item(
            target_course_id.make_usage_key('conditional', 'condone')
        )
        self.assertIsNotNone(conditional_module)
        different_course_id = SlashSeparatedCourseKey('edX', 'different_course', 'copy_run')
        self.assertListEqual(
            [
                target_course_id.make_usage_key('problem', 'choiceprob'),
                different_course_id.make_usage_key('html', 'for_testing_import_rewrites')
            ],
            conditional_module.sources_list
        )
        self.assertListEqual(
            [
                target_course_id.make_usage_key('html', 'congrats'),
                target_course_id.make_usage_key('html', 'secret_page')
            ],
            conditional_module.show_tag_list
        )

    def test_rewrite_reference(self):
        module_store = modulestore('direct')
        target_course_id = SlashSeparatedCourseKey('testX', 'peergrading_copy', 'copy_run')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['open_ended'],
            target_course_id=target_course_id
        )
        peergrading_module = module_store.get_item(
            target_course_id.make_usage_key('peergrading', 'PeerGradingLinked')
        )
        self.assertIsNotNone(peergrading_module)
        self.assertEqual(
            target_course_id.make_usage_key('combinedopenended', 'SampleQuestion'),
            peergrading_module.link_to_location
        )

    def test_rewrite_reference_value_dict(self):
        module_store = modulestore('direct')
        target_course_id = SlashSeparatedCourseKey('testX', 'split_test_copy', 'copy_run')
        import_from_xml(
            module_store,
            'common/test/data/',
            ['split_test_module'],
            target_course_id=target_course_id
        )
        split_test_module = module_store.get_item(
            target_course_id.make_usage_key('split_test', 'split1')
        )
        self.assertIsNotNone(split_test_module)
        self.assertEqual(
            {
                "0": target_course_id.make_usage_key('vertical', 'sample_0'),
                "2": target_course_id.make_usage_key('vertical', 'sample_2'),
            },
            split_test_module.group_id_to_child,
        )

########NEW FILE########
__FILENAME__ = test_import_draft_order
from django.test.utils import override_settings

from xmodule.modulestore.xml_importer import import_from_xml

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore

from contentstore.tests.modulestore_config import TEST_MODULESTORE


# This test is in the CMS module because the test configuration to use a draft
# modulestore is dependent on django.
@override_settings(MODULESTORE=TEST_MODULESTORE)
class DraftReorderTestCase(ModuleStoreTestCase):

    def test_order(self):
        store = modulestore('direct')
        draft_store = modulestore('default')
        _, course_items = import_from_xml(store, 'common/test/data/', ['import_draft_order'], draft_store=draft_store)
        course_key = course_items[0].id
        sequential = draft_store.get_item(course_key.make_usage_key('sequential', '0f4f7649b10141b0bdc9922dcf94515a'))
        verticals = sequential.children

        # The order that files are read in from the file system is not guaranteed (cannot rely on
        # alphabetical ordering, for example). Therefore, I have added a lot of variation in filename and desired
        # ordering so that the test reliably failed with the bug, at least on Linux.
        #
        # 'a', 'b', 'c', 'd', and 'z' are all drafts, with 'index_in_children_list' of
        #  2 ,  4 ,  6 ,  5 , and  0  respectively.
        #
        # '5a05be9d59fc4bb79282c94c9e6b88c7' and 'second' are public verticals.
        self.assertEqual(7, len(verticals))
        self.assertEqual(course_key.make_usage_key('vertical', 'z'), verticals[0])
        self.assertEqual(course_key.make_usage_key('vertical', '5a05be9d59fc4bb79282c94c9e6b88c7'), verticals[1])
        self.assertEqual(course_key.make_usage_key('vertical', 'a'), verticals[2])
        self.assertEqual(course_key.make_usage_key('vertical', 'second'), verticals[3])
        self.assertEqual(course_key.make_usage_key('vertical', 'b'), verticals[4])
        self.assertEqual(course_key.make_usage_key('vertical', 'd'), verticals[5])
        self.assertEqual(course_key.make_usage_key('vertical', 'c'), verticals[6])

        # Now also test that the verticals in a second sequential are correct.
        sequential = draft_store.get_item(course_key.make_usage_key('sequential', 'secondseq'))
        verticals = sequential.children
        # 'asecond' and 'zsecond' are drafts with 'index_in_children_list' 0 and 2, respectively.
        # 'secondsubsection' is a public vertical.
        self.assertEqual(3, len(verticals))
        self.assertEqual(course_key.make_usage_key('vertical', 'asecond'), verticals[0])
        self.assertEqual(course_key.make_usage_key('vertical', 'secondsubsection'), verticals[1])
        self.assertEqual(course_key.make_usage_key('vertical', 'zsecond'), verticals[2])

########NEW FILE########
__FILENAME__ = test_import_pure_xblock
"""
Integration tests for importing courses containing pure XBlocks.
"""

from django.test.utils import override_settings

from xblock.core import XBlock
from xblock.fields import String

from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.draft import as_draft
from contentstore.tests.modulestore_config import TEST_MODULESTORE


class StubXBlock(XBlock):
    """
    Stub XBlock to use in tests.

    The default XBlock implementation will load this XBlock
    from XML, using the lowercase version of the class
    as an element name ("stubxblock") and the field names
    as attributes of that element.

    Example:
        <stubxblock test_field="this is only a test" />
    """
    test_field = String(default="default")


@override_settings(MODULESTORE=TEST_MODULESTORE)
class XBlockImportTest(ModuleStoreTestCase):

    def setUp(self):
        self.store = modulestore('direct')
        self.draft_store = modulestore('default')

    @XBlock.register_temp_plugin(StubXBlock)
    def test_import_public(self):
        self._assert_import(
            'pure_xblock_public',
            'set by xml'
        )

    @XBlock.register_temp_plugin(StubXBlock)
    def test_import_draft(self):
        self._assert_import(
            'pure_xblock_draft',
            'set by xml',
            has_draft=True
        )

    def _assert_import(self, course_dir, expected_field_val, has_draft=False):
        """
        Import a course from XML, then verify that the XBlock was loaded
        with the correct field value.

        Args:
            course_dir (str): The name of the course directory (relative to the test data directory)
            expected_xblock_loc (str): The location of the XBlock in the course.
            expected_field_val (str): The expected value of the XBlock's test field.

        Kwargs:
            has_draft (bool): If true, check that a draft of the XBlock exists with
                the expected field value set.

        """
        _, courses = import_from_xml(
            self.store, 'common/test/data', [course_dir],
            draft_store=self.draft_store
        )

        xblock_location = courses[0].id.make_usage_key('stubxblock', 'xblock_test')

        if has_draft:
            xblock_location = as_draft(xblock_location)

        xblock = self.store.get_item(xblock_location)
        self.assertTrue(isinstance(xblock, StubXBlock))
        self.assertEqual(xblock.test_field, expected_field_val)

        if has_draft:
            draft_xblock = self.draft_store.get_item(xblock_location)
            self.assertTrue(isinstance(draft_xblock, StubXBlock))
            self.assertEqual(draft_xblock.test_field, expected_field_val)

########NEW FILE########
__FILENAME__ = test_orphan
"""
Test finding orphans via the view and django config
"""
import json
from contentstore.tests.utils import CourseTestCase
from student.models import CourseEnrollment
from xmodule.modulestore.django import modulestore
from contentstore.utils import reverse_course_url


class TestOrphan(CourseTestCase):
    """
    Test finding orphans via view and django config
    """
    def setUp(self):
        super(TestOrphan, self).setUp()

        runtime = self.course.runtime

        self._create_item('chapter', 'Chapter1', {}, {'display_name': 'Chapter 1'}, 'course', self.course.location.name, runtime)
        self._create_item('chapter', 'Chapter2', {}, {'display_name': 'Chapter 2'}, 'course', self.course.location.name, runtime)
        self._create_item('chapter', 'OrphanChapter', {}, {'display_name': 'Orphan Chapter'}, None, None, runtime)
        self._create_item('vertical', 'Vert1', {}, {'display_name': 'Vertical 1'}, 'chapter', 'Chapter1', runtime)
        self._create_item('vertical', 'OrphanVert', {}, {'display_name': 'Orphan Vertical'}, None, None, runtime)
        self._create_item('html', 'Html1', "<p>Goodbye</p>", {'display_name': 'Parented Html'}, 'vertical', 'Vert1', runtime)
        self._create_item('html', 'OrphanHtml', "<p>Hello</p>", {'display_name': 'Orphan html'}, None, None, runtime)
        self._create_item('static_tab', 'staticuno', "<p>tab</p>", {'display_name': 'Tab uno'}, None, None, runtime)
        self._create_item('about', 'overview', "<p>overview</p>", {}, None, None, runtime)
        self._create_item('course_info', 'updates', "<ol><li><h2>Sep 22</h2><p>test</p></li></ol>", {}, None, None, runtime)

        self.orphan_url = reverse_course_url('orphan_handler', self.course.id)

    def _create_item(self, category, name, data, metadata, parent_category, parent_name, runtime):
        location = self.course.location.replace(category=category, name=name)
        store = modulestore('direct')
        store.create_and_save_xmodule(location, data, metadata, runtime)
        if parent_name:
            # add child to parent in mongo
            parent_location = self.course.location.replace(category=parent_category, name=parent_name)
            parent = store.get_item(parent_location)
            parent.children.append(location)
            store.update_item(parent, self.user.id)

    def test_mongo_orphan(self):
        """
        Test that old mongo finds the orphans
        """
        orphans = json.loads(
            self.client.get(
                self.orphan_url,
                HTTP_ACCEPT='application/json'
            ).content
        )
        self.assertEqual(len(orphans), 3, "Wrong # {}".format(orphans))
        location = self.course.location.replace(category='chapter', name='OrphanChapter')
        self.assertIn(location.to_deprecated_string(), orphans)
        location = self.course.location.replace(category='vertical', name='OrphanVert')
        self.assertIn(location.to_deprecated_string(), orphans)
        location = self.course.location.replace(category='html', name='OrphanHtml')
        self.assertIn(location.to_deprecated_string(), orphans)

    def test_mongo_orphan_delete(self):
        """
        Test that old mongo deletes the orphans
        """
        self.client.delete(self.orphan_url)
        orphans = json.loads(
            self.client.get(self.orphan_url, HTTP_ACCEPT='application/json').content
        )
        self.assertEqual(len(orphans), 0, "Orphans not deleted {}".format(orphans))

    def test_not_permitted(self):
        """
        Test that auth restricts get and delete appropriately
        """
        test_user_client, test_user = self.create_non_staff_authed_user_client()
        CourseEnrollment.enroll(test_user, self.course.id)
        response = test_user_client.get(self.orphan_url)
        self.assertEqual(response.status_code, 403)
        response = test_user_client.delete(self.orphan_url)
        self.assertEqual(response.status_code, 403)

########NEW FILE########
__FILENAME__ = test_permissions
"""
Test CRUD for authorization.
"""
import copy

from django.test.utils import override_settings
from django.contrib.auth.models import User

from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.tests.utils import AjaxEnabledTestClient
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from contentstore.utils import reverse_url, reverse_course_url
from student.roles import CourseInstructorRole, CourseStaffRole
from contentstore.views.access import has_course_access
from student import auth


@override_settings(MODULESTORE=TEST_MODULESTORE)
class TestCourseAccess(ModuleStoreTestCase):
    """
    Course-based access (as opposed to access of a non-course xblock)
    """
    def setUp(self):
        """
        Create a staff user and log them in (creating the client).

        Create a pool of users w/o granting them any permissions
        """
        super(TestCourseAccess, self).setUp()
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the use so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

        # create a course via the view handler which has a different strategy for permissions than the factory
        self.course_key = SlashSeparatedCourseKey('myu', 'mydept.mycourse', 'myrun')
        course_url = reverse_url('course_handler')
        self.client.ajax_post(course_url,
            {
                'org': self.course_key.org,
                'number': self.course_key.course,
                'display_name': 'My favorite course',
                'run': self.course_key.run,
            }
        )

        self.users = self._create_users()

    def _create_users(self):
        """
        Create 8 users and return them
        """
        users = []
        for i in range(8):
            username = "user{}".format(i)
            email = "test+user{}@edx.org".format(i)
            user = User.objects.create_user(username, email, 'foo')
            user.is_active = True
            user.save()
            users.append(user)
        return users

    def tearDown(self):
        """
        Reverse the setup
        """
        self.client.logout()
        ModuleStoreTestCase.tearDown(self)

    def test_get_all_users(self):
        """
        Test getting all authors for a course where their permissions run the gamut of allowed group
        types.
        """
        # first check the course creator.has explicit access (don't use has_access as is_staff
        # will trump the actual test)
        self.assertTrue(
            CourseInstructorRole(self.course_key).has_user(self.user),
            "Didn't add creator as instructor."
        )
        users = copy.copy(self.users)
        # doesn't use role.users_with_role b/c it's verifying the roles.py behavior
        user_by_role = {}
        # add the misc users to the course in different groups
        for role in [CourseInstructorRole, CourseStaffRole]:
            user_by_role[role] = []
            # pylint: disable=protected-access
            group = role(self.course_key)
            # NOTE: this loop breaks the roles.py abstraction by purposely assigning
            # users to one of each possible groupname in order to test that has_course_access
            # and remove_user work
            user = users.pop()
            group.add_users(user)
            user_by_role[role].append(user)
            self.assertTrue(has_course_access(user, self.course_key), "{} does not have access".format(user))

        course_team_url = reverse_course_url('course_team_handler', self.course_key)
        response = self.client.get_html(course_team_url)
        for role in [CourseInstructorRole, CourseStaffRole]:
            for user in user_by_role[role]:
                self.assertContains(response, user.email)

        # test copying course permissions
        copy_course_key = SlashSeparatedCourseKey('copyu', 'copydept.mycourse', 'myrun')
        for role in [CourseInstructorRole, CourseStaffRole]:
            auth.add_users(
                self.user,
                role(copy_course_key),
                *role(self.course_key).users_with_role()
            )
        # verify access in copy course and verify that removal from source course w/ the various
        # groupnames works
        for role in [CourseInstructorRole, CourseStaffRole]:
            for user in user_by_role[role]:
                # forcefully decache the groups: premise is that any real request will not have
                # multiple objects repr the same user but this test somehow uses different instance
                # in above add_users call
                if hasattr(user, '_roles'):
                    del user._roles

                self.assertTrue(has_course_access(user, copy_course_key), "{} no copy access".format(user))
                auth.remove_users(self.user, role(self.course_key), user)
                self.assertFalse(has_course_access(user, self.course_key), "{} remove didn't work".format(user))

########NEW FILE########
__FILENAME__ = test_request_event
"""Tests for CMS's requests to logs"""
import mock

from django.test import TestCase
from django.core.urlresolvers import reverse
from contentstore.views.helpers import event as cms_user_track


class CMSLogTest(TestCase):
    """
    Tests that request to logs from CMS return 204s
    """

    def test_post_answers_to_log(self):
        """
        Checks that student answer requests submitted to cms's "/event" url
        via POST are correctly returned as 204s
        """
        requests = [
            {"event": "my_event", "event_type": "my_event_type", "page": "my_page"},
            {"event": "{'json': 'object'}", "event_type": unichr(512), "page": "my_page"}
        ]
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_SQL_TRACKING_LOGS': True}):
            for request_params in requests:
                response = self.client.post(reverse(cms_user_track), request_params)
                self.assertEqual(response.status_code, 204)

    def test_get_answers_to_log(self):
        """
        Checks that student answer requests submitted to cms's "/event" url
        via GET are correctly returned as 204s
        """
        requests = [
            {"event": "my_event", "event_type": "my_event_type", "page": "my_page"},
            {"event": "{'json': 'object'}", "event_type": unichr(512), "page": "my_page"}
        ]
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_SQL_TRACKING_LOGS': True}):
            for request_params in requests:
                response = self.client.get(reverse(cms_user_track), request_params)
                self.assertEqual(response.status_code, 204)

########NEW FILE########
__FILENAME__ = test_transcripts_utils
# -*- coding: utf-8 -*-
""" Tests for transcripts_utils. """
import unittest
from uuid import uuid4
import copy
import textwrap
from mock import patch, Mock

from pymongo import MongoClient

from django.test.utils import override_settings
from django.conf import settings
from django.utils import translation

from nose.plugins.skip import SkipTest

from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.exceptions import NotFoundError
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.video_module import transcripts_utils

from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


class TestGenerateSubs(unittest.TestCase):
    """Tests for `generate_subs` function."""
    def setUp(self):
        self.source_subs = {
            'start': [100, 200, 240, 390, 1000],
            'end': [200, 240, 380, 1000, 1500],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }

    def test_generate_subs_increase_speed(self):
        subs = transcripts_utils.generate_subs(2, 1, self.source_subs)
        self.assertDictEqual(
            subs,
            {
                'start': [200, 400, 480, 780, 2000],
                'end': [400, 480, 760, 2000, 3000],
                'text': ['subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5']
            }
        )

    def test_generate_subs_decrease_speed_1(self):
        subs = transcripts_utils.generate_subs(0.5, 1, self.source_subs)
        self.assertDictEqual(
            subs,
            {
                'start': [50, 100, 120, 195, 500],
                'end': [100, 120, 190, 500, 750],
                'text': ['subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5']
            }
        )

    def test_generate_subs_decrease_speed_2(self):
        """Test for correct devision during `generate_subs` process."""
        subs = transcripts_utils.generate_subs(1, 2, self.source_subs)
        self.assertDictEqual(
            subs,
            {
                'start': [50, 100, 120, 195, 500],
                'end': [100, 120, 190, 500, 750],
                'text': ['subs #1', 'subs #2', 'subs #3', 'subs #4', 'subs #5']
            }
        )


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class TestSaveSubsToStore(ModuleStoreTestCase):
    """Tests for `save_subs_to_store` function."""

    org = 'MITx'
    number = '999'
    display_name = 'Test course'

    def clear_subs_content(self):
        """Remove, if subtitles content exists."""
        try:
            content = contentstore().find(self.content_location)
            contentstore().delete(content.get_id())
        except NotFoundError:
            pass

    def setUp(self):

        self.course = CourseFactory.create(
            org=self.org, number=self.number, display_name=self.display_name)

        self.subs = {
            'start': [100, 200, 240, 390, 1000],
            'end': [200, 240, 380, 1000, 1500],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }

        self.subs_id = str(uuid4())
        filename = 'subs_{0}.srt.sjson'.format(self.subs_id)
        self.content_location = StaticContent.compute_location(self.course.id, filename)

        # incorrect  subs
        self.unjsonable_subs = set([1])  # set can't be serialized

        self.unjsonable_subs_id = str(uuid4())
        filename_unjsonable = 'subs_{0}.srt.sjson'.format(self.unjsonable_subs_id)
        self.content_location_unjsonable = StaticContent.compute_location(self.course.id, filename_unjsonable)

        self.clear_subs_content()

    def test_save_subs_to_store(self):
        with self.assertRaises(NotFoundError):
            contentstore().find(self.content_location)

        result_location = transcripts_utils.save_subs_to_store(
            self.subs,
            self.subs_id,
            self.course)

        self.assertTrue(contentstore().find(self.content_location))
        self.assertEqual(result_location, self.content_location)

    def test_save_unjsonable_subs_to_store(self):
        """
        Assures that subs, that can't be dumped, can't be found later.
        """
        with self.assertRaises(NotFoundError):
            contentstore().find(self.content_location_unjsonable)

        with self.assertRaises(TypeError):
            transcripts_utils.save_subs_to_store(
                self.unjsonable_subs,
                self.unjsonable_subs_id,
                self.course)

        with self.assertRaises(NotFoundError):
            contentstore().find(self.content_location_unjsonable)

    def tearDown(self):
        self.clear_subs_content()
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class TestDownloadYoutubeSubs(ModuleStoreTestCase):
    """Tests for `download_youtube_subs` function."""

    org = 'MITx'
    number = '999'
    display_name = 'Test course'

    def clear_subs_content(self, youtube_subs):
        """Remove, if subtitles content exists."""
        for subs_id in youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(self.course.id, filename)
            try:
                content = contentstore().find(content_location)
                contentstore().delete(content.get_id())
            except NotFoundError:
                pass

    def setUp(self):
        self.course = CourseFactory.create(
            org=self.org, number=self.number, display_name=self.display_name)

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_success_downloading_subs(self):

        response = textwrap.dedent("""<?xml version="1.0" encoding="utf-8" ?>
                <transcript>
                    <text start="0" dur="0.27"></text>
                    <text start="0.27" dur="2.45">Test text 1.</text>
                    <text start="2.72">Test text 2.</text>
                    <text start="5.43" dur="1.73">Test text 3.</text>
                </transcript>
        """)
        good_youtube_subs = {
            0.5: 'good_id_1',
            1.0: 'good_id_2',
            2.0: 'good_id_3'
        }
        self.clear_subs_content(good_youtube_subs)

        with patch('xmodule.video_module.transcripts_utils.requests.get') as mock_get:
            mock_get.return_value = Mock(status_code=200, text=response, content=response)
            # Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown
            transcripts_utils.download_youtube_subs(good_youtube_subs, self.course, settings)

        mock_get.assert_any_call('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_1'})
        mock_get.assert_any_call('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_2'})
        mock_get.assert_any_call('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_id_3'})

        # Check assets status after importing subtitles.
        for subs_id in good_youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(self.course.id, filename)
            self.assertTrue(contentstore().find(content_location))

        self.clear_subs_content(good_youtube_subs)

    def test_subs_for_html5_vid_with_periods(self):
        """
        This is to verify a fix whereby subtitle files uploaded against
        a HTML5 video that contains periods in the name causes
        incorrect subs name parsing
        """
        html5_ids = transcripts_utils.get_html5_ids(['foo.mp4', 'foo.1.bar.mp4', 'foo/bar/baz.1.4.mp4', 'foo'])
        self.assertEqual(4, len(html5_ids))
        self.assertEqual(html5_ids[0], 'foo')
        self.assertEqual(html5_ids[1], 'foo.1.bar')
        self.assertEqual(html5_ids[2], 'baz.1.4')
        self.assertEqual(html5_ids[3], 'foo')

    @patch('xmodule.video_module.transcripts_utils.requests.get')
    def test_fail_downloading_subs(self, mock_get):

        mock_get.return_value = Mock(status_code=404, text='Error 404')

        bad_youtube_subs = {
            0.5: 'BAD_YOUTUBE_ID1',
            1.0: 'BAD_YOUTUBE_ID2',
            2.0: 'BAD_YOUTUBE_ID3'
        }
        self.clear_subs_content(bad_youtube_subs)
        with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
            transcripts_utils.download_youtube_subs(bad_youtube_subs, self.course, settings)

        # Check assets status after importing subtitles.
        for subs_id in bad_youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(
                self.course.id, filename
            )
            with self.assertRaises(NotFoundError):
                contentstore().find(content_location)

        self.clear_subs_content(bad_youtube_subs)

    def test_success_downloading_chinese_transcripts(self):

        # Disabled 11/14/13
        # This test is flakey because it performs an HTTP request on an external service
        # Re-enable when `requests.get` is patched using `mock.patch`
        raise SkipTest

        good_youtube_subs = {
            1.0: 'j_jEn79vS3g',  # Chinese, utf-8
        }
        self.clear_subs_content(good_youtube_subs)

        # Check transcripts_utils.GetTranscriptsFromYouTubeException not thrown
        transcripts_utils.download_youtube_subs(good_youtube_subs, self.course, settings)

        # Check assets status after importing subtitles.
        for subs_id in good_youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(
                self.course.id, filename
            )
            self.assertTrue(contentstore().find(content_location))

        self.clear_subs_content(good_youtube_subs)


class TestGenerateSubsFromSource(TestDownloadYoutubeSubs):
    """Tests for `generate_subs_from_source` function."""

    def test_success_generating_subs(self):
        youtube_subs = {
            0.5: 'JMD_ifUUfsU',
            1.0: 'hI10vDNYz4M',
            2.0: 'AKqURZnYqpk'
        }
        srt_filedata = textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """)
        self.clear_subs_content(youtube_subs)

        # Check transcripts_utils.TranscriptsGenerationException not thrown.
        # Also checks that uppercase file extensions are supported.
        transcripts_utils.generate_subs_from_source(youtube_subs, 'SRT', srt_filedata, self.course)

        # Check assets status after importing subtitles.
        for subs_id in youtube_subs.values():
            filename = 'subs_{0}.srt.sjson'.format(subs_id)
            content_location = StaticContent.compute_location(
                self.course.id, filename
            )
            self.assertTrue(contentstore().find(content_location))

        self.clear_subs_content(youtube_subs)

    def test_fail_bad_subs_type(self):
        youtube_subs = {
            0.5: 'JMD_ifUUfsU',
            1.0: 'hI10vDNYz4M',
            2.0: 'AKqURZnYqpk'
        }

        srt_filedata = textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """)

        with self.assertRaises(transcripts_utils.TranscriptsGenerationException) as cm:
            transcripts_utils.generate_subs_from_source(youtube_subs, 'BAD_FORMAT', srt_filedata, self.course)
        exception_message = cm.exception.message
        self.assertEqual(exception_message, "We support only SubRip (*.srt) transcripts format.")

    def test_fail_bad_subs_filedata(self):
        youtube_subs = {
            0.5: 'JMD_ifUUfsU',
            1.0: 'hI10vDNYz4M',
            2.0: 'AKqURZnYqpk'
        }

        srt_filedata = """BAD_DATA"""

        with self.assertRaises(transcripts_utils.TranscriptsGenerationException) as cm:
            transcripts_utils.generate_subs_from_source(youtube_subs, 'srt', srt_filedata, self.course)
        exception_message = cm.exception.message
        self.assertEqual(exception_message, "Something wrong with SubRip transcripts file during parsing.")


class TestGenerateSrtFromSjson(TestDownloadYoutubeSubs):
    """Tests for `generate_srt_from_sjson` function."""

    def test_success_generating_subs(self):
        sjson_subs = {
            'start': [100, 200, 240, 390, 54000],
            'end': [200, 240, 380, 1000, 78400],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 1)
        self.assertTrue(srt_subs)
        expected_subs = [
            '00:00:00,100 --> 00:00:00,200\nsubs #1',
            '00:00:00,200 --> 00:00:00,240\nsubs #2',
            '00:00:00,240 --> 00:00:00,380\nsubs #3',
            '00:00:00,390 --> 00:00:01,000\nsubs #4',
            '00:00:54,000 --> 00:01:18,400\nsubs #5',
        ]

        for sub in expected_subs:
            self.assertIn(sub, srt_subs)

    def test_success_generating_subs_speed_up(self):
        sjson_subs = {
            'start': [100, 200, 240, 390, 54000],
            'end': [200, 240, 380, 1000, 78400],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 0.5)
        self.assertTrue(srt_subs)
        expected_subs = [
            '00:00:00,050 --> 00:00:00,100\nsubs #1',
            '00:00:00,100 --> 00:00:00,120\nsubs #2',
            '00:00:00,120 --> 00:00:00,190\nsubs #3',
            '00:00:00,195 --> 00:00:00,500\nsubs #4',
            '00:00:27,000 --> 00:00:39,200\nsubs #5',
        ]
        for sub in expected_subs:
            self.assertIn(sub, srt_subs)

    def test_success_generating_subs_speed_down(self):
        sjson_subs = {
            'start': [100, 200, 240, 390, 54000],
            'end': [200, 240, 380, 1000, 78400],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3',
                'subs #4',
                'subs #5'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 2)
        self.assertTrue(srt_subs)

        expected_subs = [
            '00:00:00,200 --> 00:00:00,400\nsubs #1',
            '00:00:00,400 --> 00:00:00,480\nsubs #2',
            '00:00:00,480 --> 00:00:00,760\nsubs #3',
            '00:00:00,780 --> 00:00:02,000\nsubs #4',
            '00:01:48,000 --> 00:02:36,800\nsubs #5',
        ]
        for sub in expected_subs:
            self.assertIn(sub, srt_subs)

    def test_fail_generating_subs(self):
        sjson_subs = {
            'start': [100, 200],
            'end': [100],
            'text': [
                'subs #1',
                'subs #2'
            ]
        }
        srt_subs = transcripts_utils.generate_srt_from_sjson(sjson_subs, 1)
        self.assertFalse(srt_subs)


class TestYoutubeTranscripts(unittest.TestCase):
    """
    Tests for checking right datastructure returning when using youtube api.
    """
    @patch('xmodule.video_module.transcripts_utils.requests.get')
    def test_youtube_bad_status_code(self, mock_get):
        mock_get.return_value = Mock(status_code=404, text='test')
        youtube_id = 'bad_youtube_id'
        with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
            transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation)

    @patch('xmodule.video_module.transcripts_utils.requests.get')
    def test_youtube_empty_text(self, mock_get):
        mock_get.return_value = Mock(status_code=200, text='')
        youtube_id = 'bad_youtube_id'
        with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException):
            transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation)

    def test_youtube_good_result(self):
        response = textwrap.dedent("""<?xml version="1.0" encoding="utf-8" ?>
                <transcript>
                    <text start="0" dur="0.27"></text>
                    <text start="0.27" dur="2.45">Test text 1.</text>
                    <text start="2.72">Test text 2.</text>
                    <text start="5.43" dur="1.73">Test text 3.</text>
                </transcript>
        """)
        expected_transcripts = {
            'start': [270, 2720, 5430],
            'end': [2720, 2720, 7160],
            'text': ['Test text 1.', 'Test text 2.', 'Test text 3.']
        }
        youtube_id = 'good_youtube_id'
        with patch('xmodule.video_module.transcripts_utils.requests.get') as mock_get:
            mock_get.return_value = Mock(status_code=200, text=response, content=response)
            transcripts = transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation)
        self.assertEqual(transcripts, expected_transcripts)
        mock_get.assert_called_with('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_youtube_id'})

class TestTranscript(unittest.TestCase):
    """
    Tests for Transcript class e.g. different transcript conversions.
    """
    def setUp(self):

        self.srt_transcript = textwrap.dedent("""\
            0
            00:00:10,500 --> 00:00:13,000
            Elephant&#39;s Dream

            1
            00:00:15,000 --> 00:00:18,000
            At the left we can see...

        """)


        self.sjson_transcript = textwrap.dedent("""\
            {
                "start": [
                    10500,
                    15000
                ],
                "end": [
                    13000,
                    18000
                ],
                "text": [
                    "Elephant&#39;s Dream",
                    "At the left we can see..."
                ]
            }
        """)

        self.txt_transcript = u"Elephant's Dream\nAt the left we can see..."

    def test_convert_srt_to_txt(self):
        expected = self.txt_transcript
        actual = transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'txt')
        self.assertEqual(actual, expected)

    def test_convert_srt_to_srt(self):
        expected = self.srt_transcript
        actual = transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'srt')
        self.assertEqual(actual, expected)

    def test_convert_sjson_to_txt(self):
        expected = self.txt_transcript
        actual = transcripts_utils.Transcript.convert(self.sjson_transcript, 'sjson', 'txt')
        self.assertEqual(actual, expected)

    def test_convert_sjson_to_srt(self):
        expected = self.srt_transcript
        actual = transcripts_utils.Transcript.convert(self.sjson_transcript, 'sjson', 'srt')
        self.assertEqual(actual, expected)

    def test_convert_srt_to_sjson(self):
        with self.assertRaises(NotImplementedError):
            transcripts_utils.Transcript.convert(self.srt_transcript, 'srt', 'sjson')


class TestSubsFilename(unittest.TestCase):
    """
    Tests for subs_filename funtion.
    """

    def test_unicode(self):
        name = transcripts_utils.subs_filename(u"˙∆©ƒƒƒ")
        self.assertEqual(name, u'subs_˙∆©ƒƒƒ.srt.sjson')
        name = transcripts_utils.subs_filename(u"˙∆©ƒƒƒ", 'uk')
        self.assertEqual(name, u'uk_subs_˙∆©ƒƒƒ.srt.sjson')

########NEW FILE########
__FILENAME__ = test_users_default_role
"""
Unit tests for checking default forum role "Student" of a user when he creates a course or
after deleting it creates same course again
"""
from contentstore.tests.utils import AjaxEnabledTestClient
from contentstore.utils import delete_course_and_groups, reverse_url
from courseware.tests.factories import UserFactory
from xmodule.modulestore import Location
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.locations import SlashSeparatedCourseKey

from student.models import CourseEnrollment


class TestUsersDefaultRole(ModuleStoreTestCase):
    """
    Unit tests for checking enrollment and default forum role "Student" of a logged in user
    """
    def setUp(self):
        """
        Add a user and a course
        """
        super(TestUsersDefaultRole, self).setUp()
        # create and log in a staff user.
        self.user = UserFactory(is_staff=True)  # pylint: disable=no-member
        self.client = AjaxEnabledTestClient()
        self.client.login(username=self.user.username, password='test')

        # create a course via the view handler to create course
        self.course_key = SlashSeparatedCourseKey('Org_1', 'Course_1', 'Run_1')
        self._create_course_with_given_location(self.course_key)

    def _create_course_with_given_location(self, course_key):
        """
        Create course at provided location
        """
        resp = self.client.ajax_post(
            reverse_url('course_handler'),
            {
                'org': course_key.org,
                'number': course_key.course,
                'display_name': 'test course',
                'run': course_key.run,
            }
        )
        return resp

    def tearDown(self):
        """
        Reverse the setup
        """
        self.client.logout()
        super(TestUsersDefaultRole, self).tearDown()

    def test_user_forum_default_role_on_course_deletion(self):
        """
        Test that a user enrolls and gets "Student" forum role for that course which he creates and remains
        enrolled even the course is deleted and keeps its "Student" forum role for that course
        """
        # check that user has enrollment for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        # check that user has his default "Student" forum role for this course
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

        delete_course_and_groups(self.course_key, commit=True)

        # check that user's enrollment for this course is not deleted
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        # check that user has forum role for this course even after deleting it
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

    def test_user_role_on_course_recreate(self):
        """
        Test that creating same course again after deleting it gives user his default
        forum role "Student" for that course
        """
        # check that user has enrollment and his default "Student" forum role for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

        # delete this course and recreate this course with same user
        delete_course_and_groups(self.course_key, commit=True)
        resp = self._create_course_with_given_location(self.course_key)
        self.assertEqual(resp.status_code, 200)

        # check that user has his enrollment for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        # check that user has his default "Student" forum role for this course
        self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key))  # pylint: disable=no-member

    def test_user_role_on_course_recreate_with_change_name_case(self):
        """
        Test that creating same course again with different name case after deleting it gives user
        his default forum role "Student" for that course
        """
        # check that user has enrollment and his default "Student" forum role for this course
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))
        # delete this course and recreate this course with same user
        delete_course_and_groups(self.course_key, commit=True)

        # now create same course with different name case ('uppercase')
        new_course_key = self.course_key.replace(course=self.course_key.course.upper())
        resp = self._create_course_with_given_location(new_course_key)
        self.assertEqual(resp.status_code, 200)

        # check that user has his default "Student" forum role again for this course (with changed name case)
        self.assertTrue(
            self.user.roles.filter(name="Student", course_id=new_course_key)  # pylint: disable=no-member
        )

        # Disabled due to case-sensitive test db (sqlite3)
        # # check that there user has only one "Student" forum role (with new updated course_id)
        # self.assertEqual(self.user.roles.filter(name='Student').count(), 1)  # pylint: disable=no-member
        # self.assertEqual(self.user.roles.filter(name='Student')[0].course_id, new_course_location.course_key)

########NEW FILE########
__FILENAME__ = test_utils
""" Tests for utils. """
import collections
import copy
import mock

from django.test import TestCase
from django.test.utils import override_settings

from contentstore import utils
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class LMSLinksTestCase(TestCase):
    """ Tests for LMS links. """
    def about_page_test(self):
        """ Get URL for about page, no marketing site """
        # default for ENABLE_MKTG_SITE is False.
        self.assertEquals(self.get_about_page_link(), "//localhost:8000/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'dummy-root'})
    def about_page_marketing_site_test(self):
        """ Get URL for about page, marketing root present. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//dummy-root/courses/mitX/101/test/about")
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': False}):
            self.assertEquals(self.get_about_page_link(), "//localhost:8000/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'http://www.dummy'})
    def about_page_marketing_site_remove_http_test(self):
        """ Get URL for about page, marketing root present, remove http://. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//www.dummy/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'https://www.dummy'})
    def about_page_marketing_site_remove_https_test(self):
        """ Get URL for about page, marketing root present, remove https://. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//www.dummy/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={'ROOT': 'www.dummyhttps://x'})
    def about_page_marketing_site_https__edge_test(self):
        """ Get URL for about page, only remove https:// at the beginning of the string. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), "//www.dummyhttps://x/courses/mitX/101/test/about")

    @override_settings(MKTG_URLS={})
    def about_page_marketing_urls_not_set_test(self):
        """ Error case. ENABLE_MKTG_SITE is True, but there is either no MKTG_URLS, or no MKTG_URLS Root property. """
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_MKTG_SITE': True}):
            self.assertEquals(self.get_about_page_link(), None)

    @override_settings(LMS_BASE=None)
    def about_page_no_lms_base_test(self):
        """ No LMS_BASE, nor is ENABLE_MKTG_SITE True """
        self.assertEquals(self.get_about_page_link(), None)

    def get_about_page_link(self):
        """ create mock course and return the about page link """
        course_key = SlashSeparatedCourseKey('mitX', '101', 'test')
        return utils.get_lms_link_for_about_page(course_key)

    def lms_link_test(self):
        """ Tests get_lms_link_for_item. """
        course_key = SlashSeparatedCourseKey('mitX', '101', 'test')
        location = course_key.make_usage_key('vertical', 'contacting_us')
        link = utils.get_lms_link_for_item(location, False)
        self.assertEquals(link, "//localhost:8000/courses/mitX/101/test/jump_to/i4x://mitX/101/vertical/contacting_us")

        # test preview
        link = utils.get_lms_link_for_item(location, True)
        self.assertEquals(
            link,
            "//preview/courses/mitX/101/test/jump_to/i4x://mitX/101/vertical/contacting_us"
        )

        # now test with the course' location
        location = course_key.make_usage_key('course', 'test')
        link = utils.get_lms_link_for_item(location)
        self.assertEquals(link, "//localhost:8000/courses/mitX/101/test/jump_to/i4x://mitX/101/course/test")

class ExtraPanelTabTestCase(TestCase):
    """ Tests adding and removing extra course tabs. """

    def get_tab_type_dicts(self, tab_types):
        """ Returns an array of tab dictionaries. """
        if tab_types:
            return [{'tab_type': tab_type} for tab_type in tab_types.split(',')]
        else:
            return []

    def get_course_with_tabs(self, tabs=None):
        """ Returns a mock course object with a tabs attribute. """
        if tabs is None:
            tabs = []
        course = collections.namedtuple('MockCourse', ['tabs'])
        if isinstance(tabs, basestring):
            course.tabs = self.get_tab_type_dicts(tabs)
        else:
            course.tabs = tabs
        return course

    def test_add_extra_panel_tab(self):
        """ Tests if a tab can be added to a course tab list. """
        for tab_type in utils.EXTRA_TAB_PANELS.keys():
            tab = utils.EXTRA_TAB_PANELS.get(tab_type)

            # test adding with changed = True
            for tab_setup in ['', 'x', 'x,y,z']:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = copy.copy(course.tabs)
                expected_tabs.append(tab)
                changed, actual_tabs = utils.add_extra_panel_tab(tab_type, course)
                self.assertTrue(changed)
                self.assertEqual(actual_tabs, expected_tabs)

            # test adding with changed = False
            tab_test_setup = [
                [tab],
                [tab, self.get_tab_type_dicts('x,y,z')],
                [self.get_tab_type_dicts('x,y'), tab, self.get_tab_type_dicts('z')],
                [self.get_tab_type_dicts('x,y,z'), tab]]

            for tab_setup in tab_test_setup:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = copy.copy(course.tabs)
                changed, actual_tabs = utils.add_extra_panel_tab(tab_type, course)
                self.assertFalse(changed)
                self.assertEqual(actual_tabs, expected_tabs)

    def test_remove_extra_panel_tab(self):
        """ Tests if a tab can be removed from a course tab list. """
        for tab_type in utils.EXTRA_TAB_PANELS.keys():
            tab = utils.EXTRA_TAB_PANELS.get(tab_type)

            # test removing with changed = True
            tab_test_setup = [
                [tab],
                [tab, self.get_tab_type_dicts('x,y,z')],
                [self.get_tab_type_dicts('x,y'), tab, self.get_tab_type_dicts('z')],
                [self.get_tab_type_dicts('x,y,z'), tab]]

            for tab_setup in tab_test_setup:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = [t for t in course.tabs if t != utils.EXTRA_TAB_PANELS.get(tab_type)]
                changed, actual_tabs = utils.remove_extra_panel_tab(tab_type, course)
                self.assertTrue(changed)
                self.assertEqual(actual_tabs, expected_tabs)

            # test removing with changed = False
            for tab_setup in ['', 'x', 'x,y,z']:
                course = self.get_course_with_tabs(tab_setup)
                expected_tabs = copy.copy(course.tabs)
                changed, actual_tabs = utils.remove_extra_panel_tab(tab_type, course)
                self.assertFalse(changed)
                self.assertEqual(actual_tabs, expected_tabs)


class CourseImageTestCase(TestCase):
    """Tests for course image URLs."""

    def test_get_image_url(self):
        """Test image URL formatting."""
        course = CourseFactory.create(org='edX', course='999')
        url = utils.course_image_url(course)
        self.assertEquals(url, '/c4x/edX/999/asset/{0}'.format(course.course_image))

    def test_non_ascii_image_name(self):
        # Verify that non-ascii image names are cleaned
        course = CourseFactory.create(course_image=u'before_\N{SNOWMAN}_after.jpg')
        self.assertEquals(
            utils.course_image_url(course),
            '/c4x/{org}/{course}/asset/before___after.jpg'.format(org=course.location.org, course=course.location.course)
        )

    def test_spaces_in_image_name(self):
        # Verify that image names with spaces in them are cleaned
        course = CourseFactory.create(course_image=u'before after.jpg')
        self.assertEquals(
            utils.course_image_url(course),
            '/c4x/{org}/{course}/asset/before_after.jpg'.format(
                org=course.location.org,
                course=course.location.course
            )
        )

########NEW FILE########
__FILENAME__ = utils
'''
Utilities for contentstore tests
'''

import json

from django.contrib.auth.models import User
from django.test.client import Client
from django.test.utils import override_settings

from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from contentstore.tests.modulestore_config import TEST_MODULESTORE
from contentstore.utils import get_modulestore
from student.models import Registration


def parse_json(response):
    """Parse response, which is assumed to be json"""
    return json.loads(response.content)


def user(email):
    """look up a user by email"""
    return User.objects.get(email=email)


def registration(email):
    """look up registration object by email"""
    return Registration.objects.get(user__email=email)


class AjaxEnabledTestClient(Client):
    """
    Convenience class to make testing easier.
    """
    def ajax_post(self, path, data=None, content_type="application/json", **kwargs):
        """
        Convenience method for client post which serializes the data into json and sets the accept type
        to json
        """
        if not isinstance(data, basestring):
            data = json.dumps(data or {})
        kwargs.setdefault("HTTP_X_REQUESTED_WITH", "XMLHttpRequest")
        kwargs.setdefault("HTTP_ACCEPT", "application/json")
        return self.post(path=path, data=data, content_type=content_type, **kwargs)

    def get_html(self, path, data=None, follow=False, **extra):
        """
        Convenience method for client.get which sets the accept type to html
        """
        return self.get(path, data or {}, follow, HTTP_ACCEPT="text/html", **extra)

    def get_json(self, path, data=None, follow=False, **extra):
        """
        Convenience method for client.get which sets the accept type to json
        """
        return self.get(path, data or {}, follow, HTTP_ACCEPT="application/json", **extra)



@override_settings(MODULESTORE=TEST_MODULESTORE)
class CourseTestCase(ModuleStoreTestCase):
    def setUp(self):
        """
        These tests need a user in the DB so that the django Test Client
        can log them in.
        They inherit from the ModuleStoreTestCase class so that the mongodb collection
        will be cleared out before each test case execution and deleted
        afterwards.
        """
        uname = 'testuser'
        email = 'test+courses@edx.org'
        password = 'foo'

        # Create the user so we can log them in.
        self.user = User.objects.create_user(uname, email, password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        self.user.is_active = True
        # Staff has access to view all courses
        self.user.is_staff = True
        self.user.save()

        self.client = AjaxEnabledTestClient()
        self.client.login(username=uname, password=password)

        self.course = CourseFactory.create(
            org='MITx',
            number='999',
            display_name='Robot Super Course',
        )
        self.store = get_modulestore(self.course.location)

    def create_non_staff_authed_user_client(self, authenticate=True):
        """
        Create a non-staff user, log them in (if authenticate=True), and return the client, user to use for testing.
        """
        uname = 'teststudent'
        password = 'foo'
        nonstaff = User.objects.create_user(uname, 'test+student@edx.org', password)

        # Note that we do not actually need to do anything
        # for registration if we directly mark them active.
        nonstaff.is_active = True
        nonstaff.is_staff = False
        nonstaff.save()

        client = Client()
        if authenticate:
            client.login(username=uname, password=password)
        return client, nonstaff

    def populate_course(self):
        """
        Add 2 chapters, 4 sections, 8 verticals, 16 problems to self.course (branching 2)
        """
        def descend(parent, stack):
            xblock_type = stack.pop(0)
            for _ in range(2):
                child = ItemFactory.create(category=xblock_type, parent_location=parent.location)
                if stack:
                    descend(child, stack)

        descend(self.course, ['chapter', 'sequential', 'vertical', 'problem'])

    def reload_course(self):
        """
        Reloads the course object from the database
        """
        self.course = self.store.get_course(self.course.id)

    def save_course(self):
        """
        Updates the course object in the database
        """
        self.course.save()
        self.store.update_item(self.course, self.user.id)

########NEW FILE########
__FILENAME__ = utils
# pylint: disable=E1103, E1101

import copy
import logging
import re

from django.conf import settings
from django.utils.translation import ugettext as _
from django.core.urlresolvers import reverse

from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.locations import SlashSeparatedCourseKey, Location
from xmodule.modulestore.store_utilities import delete_course
from student.roles import CourseInstructorRole, CourseStaffRole


log = logging.getLogger(__name__)

# In order to instantiate an open ended tab automatically, need to have this data
OPEN_ENDED_PANEL = {"name": _("Open Ended Panel"), "type": "open_ended"}
NOTES_PANEL = {"name": _("My Notes"), "type": "notes"}
EXTRA_TAB_PANELS = dict([(p['type'], p) for p in [OPEN_ENDED_PANEL, NOTES_PANEL]])


def delete_course_and_groups(course_id, commit=False):
    """
    This deletes the courseware associated with a course_id as well as cleaning update_item
    the various user table stuff (groups, permissions, etc.)
    """
    module_store = modulestore('direct')
    content_store = contentstore()

    module_store.ignore_write_events_on_courses.add(course_id)

    if delete_course(module_store, content_store, course_id, commit):

        print 'removing User permissions from course....'
        # in the django layer, we need to remove all the user permissions groups associated with this course
        if commit:
            try:
                staff_role = CourseStaffRole(course_id)
                staff_role.remove_users(*staff_role.users_with_role())
                instructor_role = CourseInstructorRole(course_id)
                instructor_role.remove_users(*instructor_role.users_with_role())
            except Exception as err:
                log.error("Error in deleting course groups for {0}: {1}".format(course_id, err))


def get_modulestore(category_or_location):
    """
    This function no longer does anything more than just calling `modulestore()`. It used
    to select 'direct' v 'draft' based on the category.
    """
    return modulestore()


def get_lms_link_for_item(location, preview=False):
    """
    Returns an LMS link to the course with a jump_to to the provided location.

    :param location: the location to jump to
    :param preview: True if the preview version of LMS should be returned. Default value is false.
    """
    assert(isinstance(location, Location))

    if settings.LMS_BASE is None:
        return None

    if preview:
        lms_base = settings.FEATURES.get('PREVIEW_LMS_BASE')
    else:
        lms_base = settings.LMS_BASE

    return u"//{lms_base}/courses/{course_id}/jump_to/{location}".format(
        lms_base=lms_base,
        course_id=location.course_key.to_deprecated_string(),
        location=location.to_deprecated_string(),
    )


def get_lms_link_for_about_page(course_id):
    """
    Returns the url to the course about page from the location tuple.
    """

    assert(isinstance(course_id, SlashSeparatedCourseKey))

    if settings.FEATURES.get('ENABLE_MKTG_SITE', False):
        if not hasattr(settings, 'MKTG_URLS'):
            log.exception("ENABLE_MKTG_SITE is True, but MKTG_URLS is not defined.")
            return None

        marketing_urls = settings.MKTG_URLS

        # Root will be "https://www.edx.org". The complete URL will still not be exactly correct,
        # but redirects exist from www.edx.org to get to the Drupal course about page URL.
        about_base = marketing_urls.get('ROOT', None)

        if about_base is None:
            log.exception('There is no ROOT defined in MKTG_URLS')
            return None

        # Strip off https:// (or http://) to be consistent with the formatting of LMS_BASE.
        about_base = re.sub(r"^https?://", "", about_base)

    elif settings.LMS_BASE is not None:
        about_base = settings.LMS_BASE
    else:
        return None

    return u"//{about_base_url}/courses/{course_id}/about".format(
        about_base_url=about_base,
        course_id=course_id.to_deprecated_string()
    )


def course_image_url(course):
    """Returns the image url for the course."""
    loc = StaticContent.compute_location(course.location.course_key, course.course_image)
    path = loc.to_deprecated_string()
    return path


class PublishState(object):
    """
    The publish state for a given xblock-- either 'draft', 'private', or 'public'.

    Currently in CMS, an xblock can only be in 'draft' or 'private' if it is at or below the Unit level.
    """
    draft = 'draft'
    private = 'private'
    public = 'public'


def compute_publish_state(xblock):
    """
    Returns whether this xblock is 'draft', 'public', or 'private'.

    'draft' content is in the process of being edited, but still has a previous
        version visible in the LMS
    'public' content is locked and visible in the LMS
    'private' content is editable and not visible in the LMS
    """

    if getattr(xblock, 'is_draft', False):
        try:
            modulestore('direct').get_item(xblock.location)
            return PublishState.draft
        except ItemNotFoundError:
            return PublishState.private
    else:
        return PublishState.public


def add_extra_panel_tab(tab_type, course):
    """
    Used to add the panel tab to a course if it does not exist.
    @param tab_type: A string representing the tab type.
    @param course: A course object from the modulestore.
    @return: Boolean indicating whether or not a tab was added and a list of tabs for the course.
    """
    # Copy course tabs
    course_tabs = copy.copy(course.tabs)
    changed = False
    # Check to see if open ended panel is defined in the course

    tab_panel = EXTRA_TAB_PANELS.get(tab_type)
    if tab_panel not in course_tabs:
        # Add panel to the tabs if it is not defined
        course_tabs.append(tab_panel)
        changed = True
    return changed, course_tabs


def remove_extra_panel_tab(tab_type, course):
    """
    Used to remove the panel tab from a course if it exists.
    @param tab_type: A string representing the tab type.
    @param course: A course object from the modulestore.
    @return: Boolean indicating whether or not a tab was added and a list of tabs for the course.
    """
    # Copy course tabs
    course_tabs = copy.copy(course.tabs)
    changed = False
    # Check to see if open ended panel is defined in the course

    tab_panel = EXTRA_TAB_PANELS.get(tab_type)
    if tab_panel in course_tabs:
        # Add panel to the tabs if it is not defined
        course_tabs = [ct for ct in course_tabs if ct != tab_panel]
        changed = True
    return changed, course_tabs


def reverse_url(handler_name, key_name=None, key_value=None, kwargs=None):
    """
    Creates the URL for the given handler.
    The optional key_name and key_value are passed in as kwargs to the handler.
    """
    kwargs_for_reverse = {key_name: unicode(key_value)} if key_name else None
    if kwargs:
        kwargs_for_reverse.update(kwargs)
    return reverse('contentstore.views.' + handler_name, kwargs=kwargs_for_reverse)


def reverse_course_url(handler_name, course_key, kwargs=None):
    """
    Creates the URL for handlers that use course_keys as URL parameters.
    """
    return reverse_url(handler_name, 'course_key_string', course_key, kwargs)


def reverse_usage_url(handler_name, usage_key, kwargs=None):
    """
    Creates the URL for handlers that use usage_keys as URL parameters.
    """
    return reverse_url(handler_name, 'usage_key_string', usage_key, kwargs)

########NEW FILE########
__FILENAME__ = access
from student.roles import CourseStaffRole, GlobalStaff, CourseInstructorRole
from student import auth


def has_course_access(user, course_key, role=CourseStaffRole):
    """
    Return True if user allowed to access this course_id
    Note that the CMS permissions model is with respect to courses
    There is a super-admin permissions if user.is_staff is set
    Also, since we're unifying the user database between LMS and CAS,
    I'm presuming that the course instructor (formally known as admin)
    will not be in both INSTRUCTOR and STAFF groups, so we have to cascade our
    queries here as INSTRUCTOR has all the rights that STAFF do
    """
    if GlobalStaff().has_user(user):
        return True
    return auth.has_access(user, role(course_key))


def get_user_role(user, course_id):
    """
    What type of access: staff or instructor does this user have in Studio?

    No code should use this for access control, only to quickly serialize the type of access
    where this code knows that Instructor trumps Staff and assumes the user has one or the other.

    This will not return student role because its purpose for using in Studio.

    :param course_id: the course_id of the course we're interested in
    """
    # afaik, this is only used in lti
    if auth.has_access(user, CourseInstructorRole(course_id)):
        return 'instructor'
    else:
        return 'staff'

########NEW FILE########
__FILENAME__ = assets
import logging
from functools import partial
import math
import json

from django.http import HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django_future.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_POST
from django.conf import settings

from edxmako.shortcuts import render_to_response
from cache_toolbox.core import del_cached_content

from contentstore.utils import reverse_course_url
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from django.core.exceptions import PermissionDenied
from xmodule.modulestore.keys import CourseKey, AssetKey

from util.date_utils import get_default_time_display
from util.json_request import JsonResponse
from django.http import HttpResponseNotFound
from django.utils.translation import ugettext as _
from pymongo import ASCENDING, DESCENDING
from .access import has_course_access
from xmodule.modulestore.exceptions import ItemNotFoundError

__all__ = ['assets_handler']


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
def assets_handler(request, course_key_string=None, asset_key_string=None):
    """
    The restful handler for assets.
    It allows retrieval of all the assets (as an HTML page), as well as uploading new assets,
    deleting assets, and changing the "locked" state of an asset.

    GET
        html: return an html page which will show all course assets. Note that only the asset container
            is returned and that the actual assets are filled in with a client-side request.
        json: returns a page of assets. The following parameters are supported:
            page: the desired page of results (defaults to 0)
            page_size: the number of items per page (defaults to 50)
            sort: the asset field to sort by (defaults to "date_added")
            direction: the sort direction (defaults to "descending")
    POST
        json: create (or update?) an asset. The only updating that can be done is changing the lock state.
    PUT
        json: update the locked state of an asset
    DELETE
        json: delete an asset
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    response_format = request.REQUEST.get('format', 'html')
    if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            return _assets_json(request, course_key)
        else:
            asset_key = AssetKey.from_string(asset_key_string) if asset_key_string else None
            return _update_asset(request, course_key, asset_key)
    elif request.method == 'GET':  # assume html
        return _asset_index(request, course_key)
    else:
        return HttpResponseNotFound()


def _asset_index(request, course_key):
    """
    Display an editable asset library.

    Supports start (0-based index into the list of assets) and max query parameters.
    """
    course_module = modulestore().get_course(course_key)

    return render_to_response('asset_index.html', {
        'context_course': course_module,
        'asset_callback_url': reverse_course_url('assets_handler', course_key)
    })


def _assets_json(request, course_key):
    """
    Display an editable asset library.

    Supports start (0-based index into the list of assets) and max query parameters.
    """
    requested_page = int(request.REQUEST.get('page', 0))
    requested_page_size = int(request.REQUEST.get('page_size', 50))
    requested_sort = request.REQUEST.get('sort', 'date_added')
    sort_direction = DESCENDING
    if request.REQUEST.get('direction', '').lower() == 'asc':
        sort_direction = ASCENDING

    # Convert the field name to the Mongo name
    if requested_sort == 'date_added':
        requested_sort = 'uploadDate'
    elif requested_sort == 'display_name':
        requested_sort = 'displayname'
    sort = [(requested_sort, sort_direction)]

    current_page = max(requested_page, 0)
    start = current_page * requested_page_size
    assets, total_count = _get_assets_for_page(request, course_key, current_page, requested_page_size, sort)
    end = start + len(assets)

    # If the query is beyond the final page, then re-query the final page so that at least one asset is returned
    if requested_page > 0 and start >= total_count:
        current_page = int(math.floor((total_count - 1) / requested_page_size))
        start = current_page * requested_page_size
        assets, total_count = _get_assets_for_page(request, course_key, current_page, requested_page_size, sort)
        end = start + len(assets)

    asset_json = []
    for asset in assets:
        asset_id = asset['_id']
        asset_location = StaticContent.compute_location(course_key, asset_id['name'])
        # note, due to the schema change we may not have a 'thumbnail_location' in the result set
        thumbnail_location = asset.get('thumbnail_location', None)
        if thumbnail_location:
            thumbnail_location = course_key.make_asset_key('thumbnail', thumbnail_location[4])

        asset_locked = asset.get('locked', False)
        asset_json.append(_get_asset_json(asset['displayname'], asset['uploadDate'], asset_location, thumbnail_location, asset_locked))

    return JsonResponse({
        'start': start,
        'end': end,
        'page': current_page,
        'pageSize': requested_page_size,
        'totalCount': total_count,
        'assets': asset_json,
        'sort': requested_sort,
    })


def _get_assets_for_page(request, course_key, current_page, page_size, sort):
    """
    Returns the list of assets for the specified page and page size.
    """
    start = current_page * page_size

    return contentstore().get_all_content_for_course(
        course_key, start=start, maxresults=page_size, sort=sort
    )


@require_POST
@ensure_csrf_cookie
@login_required
def _upload_asset(request, course_key):
    '''
    This method allows for POST uploading of files into the course asset
    library, which will be supported by GridFS in MongoDB.
    '''
    # Does the course actually exist?!? Get anything from it to prove its
    # existence
    try:
        modulestore().get_course(course_key)
    except ItemNotFoundError:
        # no return it as a Bad Request response
        logging.error("Could not find course: %s", course_key)
        return HttpResponseBadRequest()

    # compute a 'filename' which is similar to the location formatting, we're
    # using the 'filename' nomenclature since we're using a FileSystem paradigm
    # here. We're just imposing the Location string formatting expectations to
    # keep things a bit more consistent
    upload_file = request.FILES['file']
    filename = upload_file.name
    mime_type = upload_file.content_type

    content_loc = StaticContent.compute_location(course_key, filename)

    chunked = upload_file.multiple_chunks()
    sc_partial = partial(StaticContent, content_loc, filename, mime_type)
    if chunked:
        content = sc_partial(upload_file.chunks())
        tempfile_path = upload_file.temporary_file_path()
    else:
        content = sc_partial(upload_file.read())
        tempfile_path = None

    # first let's see if a thumbnail can be created
    (thumbnail_content, thumbnail_location) = contentstore().generate_thumbnail(
            content,
            tempfile_path=tempfile_path
    )

    # delete cached thumbnail even if one couldn't be created this time (else
    # the old thumbnail will continue to show)
    del_cached_content(thumbnail_location)
    # now store thumbnail location only if we could create it
    if thumbnail_content is not None:
        content.thumbnail_location = thumbnail_location

    # then commit the content
    contentstore().save(content)
    del_cached_content(content.location)

    # readback the saved content - we need the database timestamp
    readback = contentstore().find(content.location)

    locked = getattr(content, 'locked', False)
    response_payload = {
        'asset': _get_asset_json(content.name, readback.last_modified_at, content.location, content.thumbnail_location, locked),
        'msg': _('Upload completed')
    }

    return JsonResponse(response_payload)


@require_http_methods(("DELETE", "POST", "PUT"))
@login_required
@ensure_csrf_cookie
def _update_asset(request, course_key, asset_key):
    """
    restful CRUD operations for a course asset.
    Currently only DELETE, POST, and PUT methods are implemented.

    asset_path_encoding: the odd /c4x/org/course/category/name repr of the asset (used by Backbone as the id)
    """
    if request.method == 'DELETE':
        # Make sure the item to delete actually exists.
        try:
            content = contentstore().find(asset_key)
        except NotFoundError:
            return JsonResponse(status=404)

        # ok, save the content into the trashcan
        contentstore('trashcan').save(content)

        # see if there is a thumbnail as well, if so move that as well
        if content.thumbnail_location is not None:
            # We are ignoring the value of the thumbnail_location-- we only care whether
            # or not a thumbnail has been stored, and we can now easily create the correct path.
            thumbnail_location = course_key.make_asset_key('thumbnail', asset_key.name)
            try:
                thumbnail_content = contentstore().find(thumbnail_location)
                contentstore('trashcan').save(thumbnail_content)
                # hard delete thumbnail from origin
                contentstore().delete(thumbnail_content.get_id())
                # remove from any caching
                del_cached_content(thumbnail_location)
            except:
                logging.warning('Could not delete thumbnail: %s', thumbnail_location)

        # delete the original
        contentstore().delete(content.get_id())
        # remove from cache
        del_cached_content(content.location)
        return JsonResponse()

    elif request.method in ('PUT', 'POST'):
        if 'file' in request.FILES:
            return _upload_asset(request, course_key)
        else:
            # Update existing asset
            try:
                modified_asset = json.loads(request.body)
            except ValueError:
                return HttpResponseBadRequest()
            contentstore().set_attr(asset_key, 'locked', modified_asset['locked'])
            # Delete the asset from the cache so we check the lock status the next time it is requested.
            del_cached_content(asset_key)
            return JsonResponse(modified_asset, status=201)


def _get_asset_json(display_name, date, location, thumbnail_location, locked):
    """
    Helper method for formatting the asset information to send to client.
    """
    asset_url = location.to_deprecated_string()
    external_url = settings.LMS_BASE + asset_url
    return {
        'display_name': display_name,
        'date_added': get_default_time_display(date),
        'url': asset_url,
        'external_url': external_url,
        'portable_url': StaticContent.get_static_path_from_location(location),
        'thumbnail': thumbnail_location.to_deprecated_string() if thumbnail_location is not None else None,
        'locked': locked,
        # Needed for Backbone delete/update.
        'id': unicode(location)
    }

########NEW FILE########
__FILENAME__ = checklist
import json
import copy

from util.json_request import JsonResponse
from django.http import HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django_future.csrf import ensure_csrf_cookie
from edxmako.shortcuts import render_to_response
from django.http import HttpResponseNotFound
from django.core.exceptions import PermissionDenied
from xmodule.modulestore.keys import CourseKey
from xmodule.modulestore.django import modulestore
from contentstore.utils import get_modulestore, reverse_course_url

from .access import has_course_access
from xmodule.course_module import CourseDescriptor

from django.utils.translation import ugettext

__all__ = ['checklists_handler']


# pylint: disable=unused-argument
@require_http_methods(("GET", "POST", "PUT"))
@login_required
@ensure_csrf_cookie
def checklists_handler(request, course_key_string, checklist_index=None):
    """
    The restful handler for checklists.

    GET
        html: return html page for all checklists
        json: return json representing all checklists. checklist_index is not supported for GET at this time.
    POST or PUT
        json: updates the checked state for items within a particular checklist. checklist_index is required.
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)

    json_request = 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json')
    if request.method == 'GET':
        # If course was created before checklists were introduced, copy them over
        # from the template.
        if not course_module.checklists:
            course_module.checklists = CourseDescriptor.checklists.default
            get_modulestore(course_module.location).update_item(course_module, request.user.id)

        expanded_checklists = expand_all_action_urls(course_module)
        if json_request:
            return JsonResponse(expanded_checklists)
        else:
            handler_url = reverse_course_url('checklists_handler', course_key)
            return render_to_response('checklists.html',
                                      {
                                          'handler_url': handler_url,
                                          # context_course is used by analytics
                                          'context_course': course_module,
                                          'checklists': expanded_checklists
                                      })
    elif json_request:
        # Can now assume POST or PUT because GET handled above.
        if checklist_index is not None and 0 <= int(checklist_index) < len(course_module.checklists):
            index = int(checklist_index)
            persisted_checklist = course_module.checklists[index]
            modified_checklist = json.loads(request.body)
            # Only thing the user can modify is the "checked" state.
            # We don't want to persist what comes back from the client because it will
            # include the expanded action URLs (which are non-portable).
            for item_index, item in enumerate(modified_checklist.get('items')):
                persisted_checklist['items'][item_index]['is_checked'] = item['is_checked']
            # seeming noop which triggers kvs to record that the metadata is
            # not default
            course_module.checklists = course_module.checklists
            course_module.save()
            get_modulestore(course_module.location).update_item(course_module, request.user.id)
            expanded_checklist = expand_checklist_action_url(course_module, persisted_checklist)
            return JsonResponse(localize_checklist_text(expanded_checklist))
        else:
            return HttpResponseBadRequest(
                ("Could not save checklist state because the checklist index "
                 "was out of range or unspecified."),
                content_type="text/plain"
            )
    else:
        return HttpResponseNotFound()


def expand_all_action_urls(course_module):
    """
    Gets the checklists out of the course module and expands their action urls.

    Returns a copy of the checklists with modified urls, without modifying the persisted version
    of the checklists.
    """
    expanded_checklists = []
    for checklist in course_module.checklists:
        expanded_checklists.append(localize_checklist_text(expand_checklist_action_url(course_module, checklist)))
    return expanded_checklists


def expand_checklist_action_url(course_module, checklist):
    """
    Expands the action URLs for a given checklist and returns the modified version.

    The method does a copy of the input checklist and does not modify the input argument.
    """
    expanded_checklist = copy.deepcopy(checklist)

    urlconf_map = {
        "ManageUsers": "course_team_handler",
        "CourseOutline": "course_handler",
        "SettingsDetails": "settings_handler",
        "SettingsGrading": "grading_handler",
    }

    for item in expanded_checklist.get('items'):
        action_url = item.get('action_url')
        if action_url in urlconf_map:
            item['action_url'] = reverse_course_url(urlconf_map[action_url], course_module.id)

    return expanded_checklist

def localize_checklist_text(checklist):
    """
    Localize texts for a given checklist and returns the modified version.

    The method does an in-place operation so the input checklist is modified directly.
    """
    # Localize checklist name
    checklist['short_description'] = ugettext(checklist['short_description'])

    # Localize checklist items
    for item in checklist.get('items'):
        item['short_description'] = ugettext(item['short_description'])
        item['long_description'] = ugettext(item['long_description'])
        item['action_text'] = ugettext(item['action_text']) if item['action_text'] != "" else u""

    return checklist

########NEW FILE########
__FILENAME__ = component
from __future__ import absolute_import

import json
import logging

from django.http import HttpResponseBadRequest, Http404
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_GET
from django.core.exceptions import PermissionDenied
from django.conf import settings
from xmodule.modulestore.exceptions import ItemNotFoundError
from edxmako.shortcuts import render_to_response

from util.date_utils import get_default_time_display
from xmodule.modulestore.django import modulestore

from xblock.core import XBlock
from xblock.django.request import webob_to_django_response, django_to_webob_request
from xblock.exceptions import NoSuchHandlerError
from xblock.fields import Scope
from xblock.plugin import PluginMissingError
from xblock.runtime import Mixologist

from contentstore.utils import get_lms_link_for_item, compute_publish_state, PublishState, get_modulestore
from contentstore.views.helpers import get_parent_xblock

from models.settings.course_grading import CourseGradingModel
from xmodule.modulestore.keys import UsageKey

from .access import has_course_access
from django.utils.translation import ugettext as _

__all__ = ['OPEN_ENDED_COMPONENT_TYPES',
           'ADVANCED_COMPONENT_POLICY_KEY',
           'subsection_handler',
           'unit_handler',
           'container_handler',
           'component_handler'
           ]

log = logging.getLogger(__name__)

# NOTE: unit_handler assumes this list is disjoint from ADVANCED_COMPONENT_TYPES
COMPONENT_TYPES = ['discussion', 'html', 'problem', 'video']

OPEN_ENDED_COMPONENT_TYPES = ["combinedopenended", "peergrading"]
NOTE_COMPONENT_TYPES = ['notes']

if settings.FEATURES.get('ALLOW_ALL_ADVANCED_COMPONENTS'):
    ADVANCED_COMPONENT_TYPES = sorted(set(name for name, class_ in XBlock.load_classes()) - set(COMPONENT_TYPES))
else:

    ADVANCED_COMPONENT_TYPES = [
        'annotatable',
        'textannotation',  # module for annotating text (with annotation table)
        'videoannotation',  # module for annotating video (with annotation table)
        'imageannotation',  # module for annotating image (with annotation table)
        'word_cloud',
        'graphical_slider_tool',
        'lti',
        'concept',
        'openassessment',  # edx-ora2
    ] + OPEN_ENDED_COMPONENT_TYPES + NOTE_COMPONENT_TYPES

ADVANCED_COMPONENT_CATEGORY = 'advanced'
ADVANCED_COMPONENT_POLICY_KEY = 'advanced_modules'


@require_GET
@login_required
def subsection_handler(request, usage_key_string):
    """
    The restful handler for subsection-specific requests.

    GET
        html: return html page for editing a subsection
        json: not currently supported
    """
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
        usage_key = UsageKey.from_string(usage_key_string)
        try:
            course, item, lms_link = _get_item_in_course(request, usage_key)
        except ItemNotFoundError:
            return HttpResponseBadRequest()

        preview_link = get_lms_link_for_item(usage_key, preview=True)

        # make sure that location references a 'sequential', otherwise return
        # BadRequest
        if item.location.category != 'sequential':
            return HttpResponseBadRequest()

        parent = get_parent_xblock(item)

        # remove all metadata from the generic dictionary that is presented in a
        # more normalized UI. We only want to display the XBlocks fields, not
        # the fields from any mixins that have been added
        fields = getattr(item, 'unmixed_class', item.__class__).fields

        policy_metadata = dict(
            (field.name, field.read_from(item))
            for field
            in fields.values()
            if field.name not in ['display_name', 'start', 'due', 'format'] and field.scope == Scope.settings
        )

        can_view_live = False
        subsection_units = item.get_children()
        for unit in subsection_units:
            state = compute_publish_state(unit)
            if state in (PublishState.public, PublishState.draft):
                can_view_live = True
                break

        return render_to_response(
            'edit_subsection.html',
            {
                'subsection': item,
                'context_course': course,
                'new_unit_category': 'vertical',
                'lms_link': lms_link,
                'preview_link': preview_link,
                'course_graders': json.dumps(CourseGradingModel.fetch(usage_key.course_key).graders),
                'parent_item': parent,
                'locator': usage_key,
                'policy_metadata': policy_metadata,
                'subsection_units': subsection_units,
                'can_view_live': can_view_live
            }
        )
    else:
        return HttpResponseBadRequest("Only supports html requests")


def _load_mixed_class(category):
    """
    Load an XBlock by category name, and apply all defined mixins
    """
    component_class = XBlock.load_class(category, select=settings.XBLOCK_SELECT_FUNCTION)
    mixologist = Mixologist(settings.XBLOCK_MIXINS)
    return mixologist.mix(component_class)


@require_GET
@login_required
def unit_handler(request, usage_key_string):
    """
    The restful handler for unit-specific requests.

    GET
        html: return html page for editing a unit
        json: not currently supported
    """
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):
        usage_key = UsageKey.from_string(usage_key_string)
        try:
            course, item, lms_link = _get_item_in_course(request, usage_key)
        except ItemNotFoundError:
            return HttpResponseBadRequest()

        component_templates = _get_component_templates(course)

        xblocks = item.get_children()

        # TODO (cpennington): If we share units between courses,
        # this will need to change to check permissions correctly so as
        # to pick the correct parent subsection
        containing_subsection = get_parent_xblock(item)
        containing_section = get_parent_xblock(containing_subsection)

        # cdodge hack. We're having trouble previewing drafts via jump_to redirect
        # so let's generate the link url here

        # need to figure out where this item is in the list of children as the
        # preview will need this
        index = 1
        for child in containing_subsection.get_children():
            if child.location == item.location:
                break
            index = index + 1

        preview_lms_base = settings.FEATURES.get('PREVIEW_LMS_BASE')

        preview_lms_link = (
            u'//{preview_lms_base}/courses/{org}/{course}/{course_name}/courseware/{section}/{subsection}/{index}'
        ).format(
            preview_lms_base=preview_lms_base,
            lms_base=settings.LMS_BASE,
            org=course.location.org,
            course=course.location.course,
            course_name=course.location.name,
            section=containing_section.location.name,
            subsection=containing_subsection.location.name,
            index=index
        )

        return render_to_response('unit.html', {
            'context_course': course,
            'unit': item,
            'unit_usage_key': usage_key,
            'child_usage_keys': [block.scope_ids.usage_id for block in xblocks],
            'component_templates': json.dumps(component_templates),
            'draft_preview_link': preview_lms_link,
            'published_preview_link': lms_link,
            'subsection': containing_subsection,
            'release_date': (
                get_default_time_display(containing_subsection.start)
                if containing_subsection.start is not None else None
            ),
            'section': containing_section,
            'new_unit_category': 'vertical',
            'unit_state': compute_publish_state(item),
            'published_date': (
                get_default_time_display(item.published_date)
                if item.published_date is not None else None
            ),
        })
    else:
        return HttpResponseBadRequest("Only supports html requests")


# pylint: disable=unused-argument
@require_GET
@login_required
def container_handler(request, usage_key_string):
    """
    The restful handler for container xblock requests.

    GET
        html: returns the HTML page for editing a container
        json: not currently supported
    """
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):

        usage_key = UsageKey.from_string(usage_key_string)
        try:
            course, xblock, __ = _get_item_in_course(request, usage_key)
        except ItemNotFoundError:
            return HttpResponseBadRequest()

        component_templates = _get_component_templates(course)
        ancestor_xblocks = []
        parent = get_parent_xblock(xblock)
        while parent and parent.category != 'sequential':
            ancestor_xblocks.append(parent)
            parent = get_parent_xblock(parent)
        ancestor_xblocks.reverse()

        unit = ancestor_xblocks[0] if ancestor_xblocks else None
        unit_publish_state = compute_publish_state(unit) if unit else None

        return render_to_response('container.html', {
            'context_course': course,  # Needed only for display of menus at top of page.
            'xblock': xblock,
            'unit_publish_state': unit_publish_state,
            'xblock_locator': usage_key,
            'unit': None if not ancestor_xblocks else ancestor_xblocks[0],
            'ancestor_xblocks': ancestor_xblocks,
            'component_templates': json.dumps(component_templates),
        })
    else:
        return HttpResponseBadRequest("Only supports html requests")


def _get_component_templates(course):
    """
    Returns the applicable component templates that can be used by the specified course.
    """
    def create_template_dict(name, cat, boilerplate_name=None, is_common=False):
        """
        Creates a component template dict.

        Parameters
            display_name: the user-visible name of the component
            category: the type of component (problem, html, etc.)
            boilerplate_name: name of boilerplate for filling in default values. May be None.
            is_common: True if "common" problem, False if "advanced". May be None, as it is only used for problems.

        """
        return {
            "display_name": name,
            "category": cat,
            "boilerplate_name": boilerplate_name,
            "is_common": is_common
        }

    component_display_names = {
        'discussion': _("Discussion"),
        'html': _("HTML"),
        'problem': _("Problem"),
        'video': _("Video")
    }
    advanced_component_display_names = {}

    component_templates = []
    # The component_templates array is in the order of "advanced" (if present), followed
    # by the components in the order listed in COMPONENT_TYPES.
    for category in COMPONENT_TYPES:
        templates_for_category = []
        component_class = _load_mixed_class(category)
        # add the default template with localized display name
        # TODO: Once mixins are defined per-application, rather than per-runtime,
        # this should use a cms mixed-in class. (cpennington)
        if hasattr(component_class, 'display_name'):
            display_name = _(component_class.display_name.default) if component_class.display_name.default else _('Blank')
        else:
            display_name = _('Blank')
        templates_for_category.append(create_template_dict(display_name, category))

        # add boilerplates
        if hasattr(component_class, 'templates'):
            for template in component_class.templates():
                filter_templates = getattr(component_class, 'filter_templates', None)
                if not filter_templates or filter_templates(template, course):
                    templates_for_category.append(
                        create_template_dict(
                            _(template['metadata'].get('display_name')),
                            category,
                            template.get('template_id'),
                            template['metadata'].get('markdown') is not None
                        )
                    )
        component_templates.append({
            "type": category,
            "templates": templates_for_category,
            "display_name": component_display_names[category]
            })

    # Check if there are any advanced modules specified in the course policy.
    # These modules should be specified as a list of strings, where the strings
    # are the names of the modules in ADVANCED_COMPONENT_TYPES that should be
    # enabled for the course.
    course_advanced_keys = course.advanced_modules
    advanced_component_templates = {"type": "advanced", "templates": [], "display_name": _("Advanced")}
    # Set component types according to course policy file
    if isinstance(course_advanced_keys, list):
        for category in course_advanced_keys:
            if category in ADVANCED_COMPONENT_TYPES:
                # boilerplates not supported for advanced components
                try:
                    component_class = _load_mixed_class(category)

                    if component_class.display_name.default:
                        template_display_name = _(component_class.display_name.default)
                    else:
                        template_display_name = advanced_component_display_names.get(category, category)
                    advanced_component_templates['templates'].append(
                        create_template_dict(
                            template_display_name,
                            category
                        )
                    )
                except PluginMissingError:
                    # dhm: I got this once but it can happen any time the
                    # course author configures an advanced component which does
                    # not exist on the server. This code here merely
                    # prevents any authors from trying to instantiate the
                    # non-existent component type by not showing it in the menu
                    log.warning(
                        "Advanced component %s does not exist. It will not be added to the Studio new component menu.",
                        category
                    )
                    pass
    else:
        log.error(
            "Improper format for course advanced keys! %s",
            course_advanced_keys
        )
    if len(advanced_component_templates['templates']) > 0:
        component_templates.insert(0, advanced_component_templates)

    return component_templates


@login_required
def _get_item_in_course(request, usage_key):
    """
    Helper method for getting the old location, containing course,
    item, and lms_link for a given locator.

    Verifies that the caller has permission to access this item.
    """
    course_key = usage_key.course_key

    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course = modulestore().get_course(course_key)
    item = get_modulestore(usage_key).get_item(usage_key, depth=1)
    lms_link = get_lms_link_for_item(usage_key)

    return course, item, lms_link


@login_required
def component_handler(request, usage_key_string, handler, suffix=''):
    """
    Dispatch an AJAX action to an xblock

    Args:
        usage_id: The usage-id of the block to dispatch to
        handler (str): The handler to execute
        suffix (str): The remainder of the url to be passed to the handler

    Returns:
        :class:`django.http.HttpResponse`: The response from the handler, converted to a
            django response
    """

    usage_key = UsageKey.from_string(usage_key_string)

    descriptor = get_modulestore(usage_key).get_item(usage_key)
    # Let the module handle the AJAX
    req = django_to_webob_request(request)

    try:
        resp = descriptor.handle(handler, req, suffix)

    except NoSuchHandlerError:
        log.info("XBlock %s attempted to access missing handler %r", descriptor, handler, exc_info=True)
        raise Http404

    # unintentional update to handle any side effects of handle call; so, request user didn't author
    # the change
    get_modulestore(usage_key).update_item(descriptor, None)

    return webob_to_django_response(resp)

########NEW FILE########
__FILENAME__ = course
"""
Views related to operations on course objects
"""
import json
import random
import string  # pylint: disable=W0402

from django.utils.translation import ugettext as _
from django.contrib.auth.decorators import login_required
from django_future.csrf import ensure_csrf_cookie
from django.conf import settings
from django.views.decorators.http import require_http_methods
from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.http import HttpResponseBadRequest, HttpResponseNotFound
from util.json_request import JsonResponse
from edxmako.shortcuts import render_to_response

from xmodule.error_module import ErrorDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.content import StaticContent
from xmodule.tabs import PDFTextbookTabs

from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError
from opaque_keys import InvalidKeyError
from xmodule.modulestore.locations import Location, SlashSeparatedCourseKey

from contentstore.course_info_model import get_course_updates, update_course_updates, delete_course_update
from contentstore.utils import (
    get_lms_link_for_item,
    add_extra_panel_tab,
    remove_extra_panel_tab,
    get_modulestore,
    reverse_course_url
)
from models.settings.course_details import CourseDetails, CourseSettingsEncoder

from models.settings.course_grading import CourseGradingModel
from models.settings.course_metadata import CourseMetadata
from util.json_request import expect_json
from util.string_utils import _has_non_ascii_characters

from .access import has_course_access
from .component import (
    OPEN_ENDED_COMPONENT_TYPES,
    NOTE_COMPONENT_TYPES,
    ADVANCED_COMPONENT_POLICY_KEY
)

from django_comment_common.models import assign_default_role
from django_comment_common.utils import seed_permissions_roles

from student.models import CourseEnrollment
from student.roles import CourseRole, UserBasedRole

from xmodule.modulestore.keys import CourseKey
from course_creators.views import get_course_creator_status, add_user_with_status_unrequested
from contentstore import utils
from student.roles import CourseInstructorRole, CourseStaffRole, CourseCreatorRole, GlobalStaff
from student import auth

from microsite_configuration import microsite

__all__ = ['course_info_handler', 'course_handler', 'course_info_update_handler',
           'settings_handler',
           'grading_handler',
           'advanced_settings_handler',
           'textbooks_list_handler', 'textbooks_detail_handler']


def _get_course_module(course_key, user, depth=0):
    """
    Internal method used to calculate and return the locator and course module
    for the view functions in this file.
    """
    if not has_course_access(user, course_key):
        raise PermissionDenied()
    course_module = modulestore().get_course(course_key, depth=depth)
    return course_module


# pylint: disable=unused-argument
@login_required
def course_handler(request, course_key_string=None):
    """
    The restful handler for course specific requests.
    It provides the course tree with the necessary information for identifying and labeling the parts. The root
    will typically be a 'course' object but may not be especially as we support modules.

    GET
        html: return course listing page if not given a course id
        html: return html page overview for the given course if given a course id
        json: return json representing the course branch's index entry as well as dag w/ all of the children
        replaced w/ json docs where each doc has {'_id': , 'display_name': , 'children': }
    POST
        json: create a course, return resulting json
        descriptor (same as in GET course/...). Leaving off /branch/draft would imply create the course w/ default
        branches. Cannot change the structure contents ('_id', 'display_name', 'children') but can change the
        index entry.
    PUT
        json: update this course (index entry not xblock) such as repointing head, changing display name, org,
        offering. Return same json as above.
    DELETE
        json: delete this branch from this course (leaving off /branch/draft would imply delete the course)
    """
    response_format = request.REQUEST.get('format', 'html')
    if response_format == 'json' or 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            return JsonResponse(_course_json(request, CourseKey.from_string(course_key_string)))
        elif request.method == 'POST':  # not sure if this is only post. If one will have ids, it goes after access
            return create_new_course(request)
        elif not has_course_access(request.user, CourseKey.from_string(course_key_string)):
            raise PermissionDenied()
        elif request.method == 'PUT':
            raise NotImplementedError()
        elif request.method == 'DELETE':
            raise NotImplementedError()
        else:
            return HttpResponseBadRequest()
    elif request.method == 'GET':  # assume html
        if course_key_string is None:
            return course_listing(request)
        else:
            return course_index(request, CourseKey.from_string(course_key_string))
    else:
        return HttpResponseNotFound()


@login_required
def _course_json(request, course_key):
    """
    Returns a JSON overview of a course
    """
    course_module = _get_course_module(course_key, request.user, depth=None)
    return _xmodule_json(course_module, course_module.id)


def _xmodule_json(xmodule, course_id):
    """
    Returns a JSON overview of an XModule
    """
    is_container = xmodule.has_children
    result = {
        'display_name': xmodule.display_name,
        'id': unicode(xmodule.location),
        'category': xmodule.category,
        'is_draft': getattr(xmodule, 'is_draft', False),
        'is_container': is_container,
    }
    if is_container:
        result['children'] = [_xmodule_json(child, course_id) for child in xmodule.get_children()]
    return result


def _accessible_courses_list(request):
    """
    List all courses available to the logged in user by iterating through all the courses
    """
    courses = modulestore('direct').get_courses()

    # filter out courses that we don't have access to
    def course_filter(course):
        """
        Get courses to which this user has access
        """
        if GlobalStaff().has_user(request.user):
            return course.location.course != 'templates'

        return (has_course_access(request.user, course.id)
                # pylint: disable=fixme
                # TODO remove this condition when templates purged from db
                and course.location.course != 'templates'
                )
    courses = filter(course_filter, courses)
    return courses


def _accessible_courses_list_from_groups(request):
    """
    List all courses available to the logged in user by reversing access group names
    """
    courses_list = {}

    instructor_courses = UserBasedRole(request.user, CourseInstructorRole.ROLE).courses_with_role()
    staff_courses = UserBasedRole(request.user, CourseStaffRole.ROLE).courses_with_role()
    all_courses = instructor_courses | staff_courses

    for course_access in all_courses:
        course_key = course_access.course_id
        if course_key not in courses_list:
            course = modulestore('direct').get_course(course_key)
            if course is None:
                raise ItemNotFoundError(course_key)
            courses_list[course_key] = course

    return courses_list.values()


@login_required
@ensure_csrf_cookie
def course_listing(request):
    """
    List all courses available to the logged in user
    Try to get all courses by first reversing django groups and fallback to old method if it fails
    Note: overhead of pymongo reads will increase if getting courses from django groups fails
    """
    if GlobalStaff().has_user(request.user):
        # user has global access so no need to get courses from django groups
        courses = _accessible_courses_list(request)
    else:
        try:
            courses = _accessible_courses_list_from_groups(request)
        except ItemNotFoundError:
            # user have some old groups or there was some error getting courses from django groups
            # so fallback to iterating through all courses
            courses = _accessible_courses_list(request)

    def format_course_for_view(course):
        """
        return tuple of the data which the view requires for each course
        """
        return (
            course.display_name,
            reverse_course_url('course_handler', course.id),
            get_lms_link_for_item(course.location),
            course.display_org_with_default,
            course.display_number_with_default,
            course.location.name
        )

    return render_to_response('index.html', {
        'courses': [format_course_for_view(c) for c in courses if not isinstance(c, ErrorDescriptor)],
        'user': request.user,
        'request_course_creator_url': reverse('contentstore.views.request_course_creator'),
        'course_creator_status': _get_course_creator_status(request.user),
        'allow_unicode_course_id': settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID', False)
    })


@login_required
@ensure_csrf_cookie
def course_index(request, course_key):
    """
    Display an editable course overview.

    org, course, name: Attributes of the Location for the item to edit
    """
    course_module = _get_course_module(course_key, request.user, depth=3)
    lms_link = get_lms_link_for_item(course_module.location)
    sections = course_module.get_children()


    return render_to_response('overview.html', {
        'context_course': course_module,
        'lms_link': lms_link,
        'sections': sections,
        'course_graders': json.dumps(
            CourseGradingModel.fetch(course_key).graders
        ),
        'new_section_category': 'chapter',
        'new_subsection_category': 'sequential',
        'new_unit_category': 'vertical',
        'category': 'vertical'
    })


@expect_json
def create_new_course(request):
    """
    Create a new course.

    Returns the URL for the course overview page.
    """
    if not auth.has_access(request.user, CourseCreatorRole()):
        raise PermissionDenied()

    org = request.json.get('org')
    number = request.json.get('number')
    display_name = request.json.get('display_name')
    run = request.json.get('run')

    # allow/disable unicode characters in course_id according to settings
    if not settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID'):
        if _has_non_ascii_characters(org) or _has_non_ascii_characters(number) or _has_non_ascii_characters(run):
            return JsonResponse(
                {'error': _('Special characters not allowed in organization, course number, and course run.')},
                status=400
            )

    try:
        course_key = SlashSeparatedCourseKey(org, number, run)

        # instantiate the CourseDescriptor and then persist it
        # note: no system to pass
        if display_name is None:
            metadata = {}
        else:
            metadata = {'display_name': display_name}

        # Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for
        # existing xml courses this cannot be changed in CourseDescriptor.
        # # TODO get rid of defining wiki slug in this org/course/run specific way and reconcile
        # w/ xmodule.course_module.CourseDescriptor.__init__
        wiki_slug = u"{0}.{1}.{2}".format(course_key.org, course_key.course, course_key.run)
        definition_data = {'wiki_slug': wiki_slug}

        # Create the course then fetch it from the modulestore
        # Check if role permissions group for a course named like this already exists
        # Important because role groups are case insensitive
        if CourseRole.course_group_already_exists(course_key):
            raise InvalidLocationError()

        fields = {}
        fields.update(definition_data)
        fields.update(metadata)

        # Creating the course raises InvalidLocationError if an existing course with this org/name is found
        new_course = modulestore('direct').create_course(
            course_key.org,
            course_key.offering,
            fields=fields,
        )

        # can't use auth.add_users here b/c it requires request.user to already have Instructor perms in this course
        # however, we can assume that b/c this user had authority to create the course, the user can add themselves
        CourseInstructorRole(new_course.id).add_users(request.user)
        auth.add_users(request.user, CourseStaffRole(new_course.id), request.user)

        # seed the forums
        seed_permissions_roles(new_course.id)

        # auto-enroll the course creator in the course so that "View Live" will
        # work.
        CourseEnrollment.enroll(request.user, new_course.id)
        _users_assign_default_role(new_course.id)

        return JsonResponse({
            'url': reverse_course_url('course_handler', new_course.id)
        })

    except InvalidLocationError:
        return JsonResponse({
            'ErrMsg': _(
                'There is already a course defined with the same '
                'organization, course number, and course run. Please '
                'change either organization or course number to be unique.'
            ),
            'OrgErrMsg': _(
                'Please change either the organization or '
                'course number so that it is unique.'),
            'CourseErrMsg': _(
                'Please change either the organization or '
                'course number so that it is unique.'),
        })
    except InvalidKeyError as error:
        return JsonResponse({
            "ErrMsg": _("Unable to create course '{name}'.\n\n{err}").format(name=display_name, err=error.message)}
        )


def _users_assign_default_role(course_id):
    """
    Assign 'Student' role to all previous users (if any) for this course
    """
    enrollments = CourseEnrollment.objects.filter(course_id=course_id)
    for enrollment in enrollments:
        assign_default_role(course_id, enrollment.user)


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(["GET"])
def course_info_handler(request, course_key_string):
    """
    GET
        html: return html for editing the course info handouts and updates.
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    if 'text/html' in request.META.get('HTTP_ACCEPT', 'text/html'):

        return render_to_response(
            'course_info.html',
            {
                'context_course': course_module,
                'updates_url': reverse_course_url('course_info_update_handler', course_key),
                'handouts_locator': course_key.make_usage_key('course_info', 'handouts'),
                'base_asset_url': StaticContent.get_base_url_path_for_course_assets(course_module.id)
            }
        )
    else:
        return HttpResponseBadRequest("Only supports html requests")


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
@expect_json
def course_info_update_handler(request, course_key_string, provided_id=None):
    """
    restful CRUD operations on course_info updates.
    provided_id should be none if it's new (create) and index otherwise.
    GET
        json: return the course info update models
    POST
        json: create an update
    PUT or DELETE
        json: change an existing update
    """
    if 'application/json' not in request.META.get('HTTP_ACCEPT', 'application/json'):
        return HttpResponseBadRequest("Only supports json requests")

    course_key = CourseKey.from_string(course_key_string)
    usage_key = course_key.make_usage_key('course_info', 'updates')
    if provided_id == '':
        provided_id = None

    # check that logged in user has permissions to this item (GET shouldn't require this level?)
    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    if request.method == 'GET':
        course_updates = get_course_updates(usage_key, provided_id)
        if isinstance(course_updates, dict) and course_updates.get('error'):
            return JsonResponse(course_updates, course_updates.get('status', 400))
        else:
            return JsonResponse(course_updates)
    elif request.method == 'DELETE':
        try:
            return JsonResponse(delete_course_update(usage_key, request.json, provided_id, request.user))
        except:
            return HttpResponseBadRequest(
                "Failed to delete",
                content_type="text/plain"
            )
    # can be either and sometimes django is rewriting one to the other:
    elif request.method in ('POST', 'PUT'):
        try:
            return JsonResponse(update_course_updates(usage_key, request.json, provided_id, request.user))
        except:
            return HttpResponseBadRequest(
                "Failed to save",
                content_type="text/plain"
            )


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "PUT", "POST"))
@expect_json
def settings_handler(request, course_key_string):
    """
    Course settings for dates and about pages
    GET
        html: get the page
        json: get the CourseDetails model
    PUT
        json: update the Course and About xblocks through the CourseDetails model
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
        upload_asset_url = reverse_course_url('assets_handler', course_key)

        # see if the ORG of this course can be attributed to a 'Microsite'. In that case, the
        # course about page should be editable in Studio
        about_page_editable = not microsite.get_value_for_org(
            course_module.location.org,
            'ENABLE_MKTG_SITE',
            settings.FEATURES.get('ENABLE_MKTG_SITE', False)
        )

        short_description_editable = settings.FEATURES.get('EDITABLE_SHORT_DESCRIPTION', True)

        return render_to_response('settings.html', {
            'context_course': course_module,
            'course_locator': course_key,
            'lms_link_for_about_page': utils.get_lms_link_for_about_page(course_key),
            'course_image_url': utils.course_image_url(course_module),
            'details_url': reverse_course_url('settings_handler', course_key),
            'about_page_editable': about_page_editable,
            'short_description_editable': short_description_editable,
            'upload_asset_url': upload_asset_url
        })
    elif 'application/json' in request.META.get('HTTP_ACCEPT', ''):
        if request.method == 'GET':
            return JsonResponse(
                CourseDetails.fetch(course_key),
                # encoder serializes dates, old locations, and instances
                encoder=CourseSettingsEncoder
            )
        else:  # post or put, doesn't matter.
            return JsonResponse(
                CourseDetails.update_from_json(course_key, request.json, request.user),
                encoder=CourseSettingsEncoder
            )


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
@expect_json
def grading_handler(request, course_key_string, grader_index=None):
    """
    Course Grading policy configuration
    GET
        html: get the page
        json no grader_index: get the CourseGrading model (graceperiod, cutoffs, and graders)
        json w/ grader_index: get the specific grader
    PUT
        json no grader_index: update the Course through the CourseGrading model
        json w/ grader_index: create or update the specific grader (create if index out of range)
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)

    if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':
        course_details = CourseGradingModel.fetch(course_key)

        return render_to_response('settings_graders.html', {
            'context_course': course_module,
            'course_locator': course_key,
            'course_details': json.dumps(course_details, cls=CourseSettingsEncoder),
            'grading_url': reverse_course_url('grading_handler', course_key),
        })
    elif 'application/json' in request.META.get('HTTP_ACCEPT', ''):
        if request.method == 'GET':
            if grader_index is None:
                return JsonResponse(
                    CourseGradingModel.fetch(course_key),
                    # encoder serializes dates, old locations, and instances
                    encoder=CourseSettingsEncoder
                )
            else:
                return JsonResponse(CourseGradingModel.fetch_grader(course_key, grader_index))
        elif request.method in ('POST', 'PUT'):  # post or put, doesn't matter.
            # None implies update the whole model (cutoffs, graceperiod, and graders) not a specific grader
            if grader_index is None:
                return JsonResponse(
                    CourseGradingModel.update_from_json(course_key, request.json, request.user),
                    encoder=CourseSettingsEncoder
                )
            else:
                return JsonResponse(
                    CourseGradingModel.update_grader_from_json(course_key, request.json, request.user)
                )
        elif request.method == "DELETE" and grader_index is not None:
            CourseGradingModel.delete_grader(course_key, grader_index, request.user)
            return JsonResponse()


# pylint: disable=invalid-name
def _config_course_advanced_components(request, course_module):
    """
    Check to see if the user instantiated any advanced components. This
    is a hack that does the following :
    1) adds/removes the open ended panel tab to a course automatically
    if the user has indicated that they want to edit the
    combinedopendended or peergrading module
    2) adds/removes the notes panel tab to a course automatically if
    the user has indicated that they want the notes module enabled in
    their course
    """
    # TODO refactor the above into distinct advanced policy settings
    filter_tabs = True  # Exceptional conditions will pull this to False
    if ADVANCED_COMPONENT_POLICY_KEY in request.json:  # Maps tab types to components
        tab_component_map = {
            'open_ended': OPEN_ENDED_COMPONENT_TYPES,
            'notes': NOTE_COMPONENT_TYPES,
        }
        # Check to see if the user instantiated any notes or open ended components
        for tab_type in tab_component_map.keys():
            component_types = tab_component_map.get(tab_type)
            found_ac_type = False
            for ac_type in component_types:
                if ac_type in request.json[ADVANCED_COMPONENT_POLICY_KEY]:
                    # Add tab to the course if needed
                    changed, new_tabs = add_extra_panel_tab(tab_type, course_module)
                    # If a tab has been added to the course, then send the
                    # metadata along to CourseMetadata.update_from_json
                    if changed:
                        course_module.tabs = new_tabs
                        request.json.update({'tabs': new_tabs})
                        # Indicate that tabs should not be filtered out of
                        # the metadata
                        filter_tabs = False  # Set this flag to avoid the tab removal code below.
                    found_ac_type = True  #break

            # If we did not find a module type in the advanced settings,
            # we may need to remove the tab from the course.
            if not found_ac_type:  # Remove tab from the course if needed
                changed, new_tabs = remove_extra_panel_tab(tab_type, course_module)
                if changed:
                    course_module.tabs = new_tabs
                    request.json.update({'tabs':new_tabs})
                    # Indicate that tabs should *not* be filtered out of
                    # the metadata
                    filter_tabs = False

    return filter_tabs


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
@expect_json
def advanced_settings_handler(request, course_key_string):
    """
    Course settings configuration
    GET
        html: get the page
        json: get the model
    PUT, POST
        json: update the Course's settings. The payload is a json rep of the
            metadata dicts. The dict can include a "unsetKeys" entry which is a list
            of keys whose values to unset: i.e., revert to default
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    if 'text/html' in request.META.get('HTTP_ACCEPT', '') and request.method == 'GET':

        return render_to_response('settings_advanced.html', {
            'context_course': course_module,
            'advanced_dict': json.dumps(CourseMetadata.fetch(course_module)),
            'advanced_settings_url': reverse_course_url('advanced_settings_handler', course_key)
        })
    elif 'application/json' in request.META.get('HTTP_ACCEPT', ''):
        if request.method == 'GET':
            return JsonResponse(CourseMetadata.fetch(course_module))
        else:
            # Whether or not to filter the tabs key out of the settings metadata
            filter_tabs = _config_course_advanced_components(request, course_module)
            try:
                return JsonResponse(CourseMetadata.update_from_json(
                    course_module,
                    request.json,
                    filter_tabs=filter_tabs,
                    user=request.user,
                ))
            except (TypeError, ValueError) as err:
                return HttpResponseBadRequest(
                    "Incorrect setting format. {}".format(err),
                    content_type="text/plain"
                )


class TextbookValidationError(Exception):
    "An error thrown when a textbook input is invalid"
    pass


def validate_textbooks_json(text):
    """
    Validate the given text as representing a single PDF textbook
    """
    try:
        textbooks = json.loads(text)
    except ValueError:
        raise TextbookValidationError("invalid JSON")
    if not isinstance(textbooks, (list, tuple)):
        raise TextbookValidationError("must be JSON list")
    for textbook in textbooks:
        validate_textbook_json(textbook)
    # check specified IDs for uniqueness
    all_ids = [textbook["id"] for textbook in textbooks if "id" in textbook]
    unique_ids = set(all_ids)
    if len(all_ids) > len(unique_ids):
        raise TextbookValidationError("IDs must be unique")
    return textbooks


def validate_textbook_json(textbook):
    """
    Validate the given text as representing a list of PDF textbooks
    """
    if isinstance(textbook, basestring):
        try:
            textbook = json.loads(textbook)
        except ValueError:
            raise TextbookValidationError("invalid JSON")
    if not isinstance(textbook, dict):
        raise TextbookValidationError("must be JSON object")
    if not textbook.get("tab_title"):
        raise TextbookValidationError("must have tab_title")
    tid = unicode(textbook.get("id", ""))
    if tid and not tid[0].isdigit():
        raise TextbookValidationError("textbook ID must start with a digit")
    return textbook


def assign_textbook_id(textbook, used_ids=()):
    """
    Return an ID that can be assigned to a textbook
    and doesn't match the used_ids
    """
    tid = Location.clean(textbook["tab_title"])
    if not tid[0].isdigit():
        # stick a random digit in front
        tid = random.choice(string.digits) + tid
    while tid in used_ids:
        # add a random ASCII character to the end
        tid = tid + random.choice(string.ascii_lowercase)
    return tid


@require_http_methods(("GET", "POST", "PUT"))
@login_required
@ensure_csrf_cookie
def textbooks_list_handler(request, course_key_string):
    """
    A RESTful handler for textbook collections.

    GET
        html: return textbook list page (Backbone application)
        json: return JSON representation of all textbooks in this course
    POST
        json: create a new textbook for this course
    PUT
        json: overwrite all textbooks in the course with the given list
    """
    course_key = CourseKey.from_string(course_key_string)
    course = _get_course_module(course_key, request.user)
    store = get_modulestore(course.location)

    if not "application/json" in request.META.get('HTTP_ACCEPT', 'text/html'):
        # return HTML page
        upload_asset_url = reverse_course_url('assets_handler', course_key)
        textbook_url = reverse_course_url('textbooks_list_handler', course_key)
        return render_to_response('textbooks.html', {
            'context_course': course,
            'textbooks': course.pdf_textbooks,
            'upload_asset_url': upload_asset_url,
            'textbook_url': textbook_url,
        })

    # from here on down, we know the client has requested JSON
    if request.method == 'GET':
        return JsonResponse(course.pdf_textbooks)
    elif request.method == 'PUT':
        try:
            textbooks = validate_textbooks_json(request.body)
        except TextbookValidationError as err:
            return JsonResponse({"error": err.message}, status=400)

        tids = set(t["id"] for t in textbooks if "id" in t)
        for textbook in textbooks:
            if not "id" in textbook:
                tid = assign_textbook_id(textbook, tids)
                textbook["id"] = tid
                tids.add(tid)

        if not any(tab['type'] == PDFTextbookTabs.type for tab in course.tabs):
            course.tabs.append(PDFTextbookTabs())
        course.pdf_textbooks = textbooks
        store.update_item(course, request.user.id)
        return JsonResponse(course.pdf_textbooks)
    elif request.method == 'POST':
        # create a new textbook for the course
        try:
            textbook = validate_textbook_json(request.body)
        except TextbookValidationError as err:
            return JsonResponse({"error": err.message}, status=400)
        if not textbook.get("id"):
            tids = set(t["id"] for t in course.pdf_textbooks if "id" in t)
            textbook["id"] = assign_textbook_id(textbook, tids)
        existing = course.pdf_textbooks
        existing.append(textbook)
        course.pdf_textbooks = existing
        if not any(tab['type'] == PDFTextbookTabs.type for tab in course.tabs):
            course.tabs.append(PDFTextbookTabs())
        store.update_item(course, request.user.id)
        resp = JsonResponse(textbook, status=201)
        resp["Location"] = reverse_course_url(
            'textbooks_detail_handler',
            course.id,
            kwargs={'textbook_id': textbook["id"]}
        )
        return resp


@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
def textbooks_detail_handler(request, course_key_string, textbook_id):
    """
    JSON API endpoint for manipulating a textbook via its internal ID.
    Used by the Backbone application.

    GET
        json: return JSON representation of textbook
    POST or PUT
        json: update textbook based on provided information
    DELETE
        json: remove textbook
    """
    course_key = CourseKey.from_string(course_key_string)
    course_module = _get_course_module(course_key, request.user)
    store = get_modulestore(course_module.location)
    matching_id = [tb for tb in course_module.pdf_textbooks
                   if unicode(tb.get("id")) == unicode(textbook_id)]
    if matching_id:
        textbook = matching_id[0]
    else:
        textbook = None

    if request.method == 'GET':
        if not textbook:
            return JsonResponse(status=404)
        return JsonResponse(textbook)
    elif request.method in ('POST', 'PUT'):  # can be either and sometimes
                                        # django is rewriting one to the other
        try:
            new_textbook = validate_textbook_json(request.body)
        except TextbookValidationError as err:
            return JsonResponse({"error": err.message}, status=400)
        new_textbook["id"] = textbook_id
        if textbook:
            i = course_module.pdf_textbooks.index(textbook)
            new_textbooks = course_module.pdf_textbooks[0:i]
            new_textbooks.append(new_textbook)
            new_textbooks.extend(course_module.pdf_textbooks[i + 1:])
            course_module.pdf_textbooks = new_textbooks
        else:
            course_module.pdf_textbooks.append(new_textbook)
        store.update_item(course_module, request.user.id)
        return JsonResponse(new_textbook, status=201)
    elif request.method == 'DELETE':
        if not textbook:
            return JsonResponse(status=404)
        i = course_module.pdf_textbooks.index(textbook)
        remaining_textbooks = course_module.pdf_textbooks[0:i]
        remaining_textbooks.extend(course_module.pdf_textbooks[i + 1:])
        course_module.pdf_textbooks = remaining_textbooks
        store.update_item(course_module, request.user.id)
        return JsonResponse()


def _get_course_creator_status(user):
    """
    Helper method for returning the course creator status for a particular user,
    taking into account the values of DISABLE_COURSE_CREATION and ENABLE_CREATOR_GROUP.

    If the user passed in has not previously visited the index page, it will be
    added with status 'unrequested' if the course creator group is in use.
    """
    if user.is_staff:
        course_creator_status = 'granted'
    elif settings.FEATURES.get('DISABLE_COURSE_CREATION', False):
        course_creator_status = 'disallowed_for_this_site'
    elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False):
        course_creator_status = get_course_creator_status(user)
        if course_creator_status is None:
            # User not grandfathered in as an existing user, has not previously visited the dashboard page.
            # Add the user to the course creator admin table with status 'unrequested'.
            add_user_with_status_unrequested(user)
            course_creator_status = get_course_creator_status(user)
    else:
        course_creator_status = 'granted'

    return course_creator_status

########NEW FILE########
__FILENAME__ = dev
"""
Views that are only activated when the project is running in development mode.
These views will NOT be shown on production: trying to access them will result
in a 404 error.
"""
# pylint: disable=W0613
from edxmako.shortcuts import render_to_response
from mako.exceptions import TopLevelLookupException
from django.http import HttpResponseNotFound


def dev_mode(request):
    "Sample static view"
    return render_to_response("dev/dev_mode.html")


def dev_show_template(request, template):
    """
    Shows the specified template as an HTML page.
    e.g. /template/ux/reference/container.html shows the template under ux/reference/container.html

    Note: dynamic parameters can also be passed to the page.
    e.g. /template/ux/reference/container.html?name=Foo
    """
    try:
        return render_to_response(template, request.GET.dict())
    except TopLevelLookupException:
        return HttpResponseNotFound("Couldn't find template {tpl}".format(tpl=template))

########NEW FILE########
__FILENAME__ = error
# pylint: disable=C0111,W0613

from django.http import (HttpResponse, HttpResponseServerError,
                         HttpResponseNotFound)
from edxmako.shortcuts import render_to_string, render_to_response
import functools
import json

__all__ = ['not_found', 'server_error', 'render_404', 'render_500']


def jsonable_error(status=500, message="The Studio servers encountered an error"):
    """
    A decorator to make an error view return an JSON-formatted message if
    it was requested via AJAX.
    """
    def outer(func):
        @functools.wraps(func)
        def inner(request, *args, **kwargs):
            if request.is_ajax():
                content = json.dumps({"error": message})
                return HttpResponse(content, content_type="application/json",
                                    status=status)
            else:
                return func(request, *args, **kwargs)
        return inner
    return outer


@jsonable_error(404, "Resource not found")
def not_found(request):
    return render_to_response('error.html', {'error': '404'})


@jsonable_error(500, "The Studio servers encountered an error")
def server_error(request):
    return render_to_response('error.html', {'error': '500'})


@jsonable_error(404, "Resource not found")
def render_404(request):
    return HttpResponseNotFound(render_to_string('404.html', {}))


@jsonable_error(500, "The Studio servers encountered an error")
def render_500(request):
    return HttpResponseServerError(render_to_string('500.html', {}))

########NEW FILE########
__FILENAME__ = export_git
"""
This views handles exporting the course xml to a git repository if
the giturl attribute is set.
"""

import logging

from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django_future.csrf import ensure_csrf_cookie
from django.utils.translation import ugettext as _

from .access import has_course_access
import contentstore.git_export_utils as git_export_utils
from edxmako.shortcuts import render_to_response
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey

log = logging.getLogger(__name__)


@ensure_csrf_cookie
@login_required
def export_git(request, course_key_string):
    """
    This method serves up the 'Export to Git' page
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)
    failed = False

    log.debug('export_git course_module=%s', course_module)

    msg = ""
    if 'action' in request.GET and course_module.giturl:
        if request.GET['action'] == 'push':
            try:
                git_export_utils.export_to_git(
                    course_module.id,
                    course_module.giturl,
                    request.user,
                )
                msg = _('Course successfully exported to git repository')
            except git_export_utils.GitExportError as ex:
                failed = True
                msg = unicode(ex)

    return render_to_response('export_git.html', {
        'context_course': course_module,
        'msg': msg,
        'failed': failed,
    })

########NEW FILE########
__FILENAME__ = helpers
import logging

from django.http import HttpResponse
from django.shortcuts import redirect
from edxmako.shortcuts import render_to_string, render_to_response
from xmodule.modulestore.django import modulestore
from contentstore.utils import reverse_course_url, reverse_usage_url

__all__ = ['edge', 'event', 'landing']

EDITING_TEMPLATES = [
    "basic-modal", "modal-button", "edit-xblock-modal", "editor-mode-button", "upload-dialog", "image-modal",
    "add-xblock-component", "add-xblock-component-button", "add-xblock-component-menu",
    "add-xblock-component-menu-problem"
]

# points to the temporary course landing page with log in and sign up
def landing(request, org, course, coursename):
    return render_to_response('temp-course-landing.html', {})


# points to the temporary edge page
def edge(request):
    return redirect('/')


def event(request):
    '''
    A noop to swallow the analytics call so that cms methods don't spook and poor developers looking at
    console logs don't get distracted :-)
    '''
    return HttpResponse(status=204)


def render_from_lms(template_name, dictionary, context=None, namespace='main'):
    """
    Render a template using the LMS MAKO_TEMPLATES
    """
    return render_to_string(template_name, dictionary, context, namespace="lms." + namespace)


def _xmodule_recurse(item, action, ignore_exception=()):
    """
    Recursively apply provided action on item and its children

    ignore_exception (Exception Object): A optional argument; when passed ignores the corresponding
        exception raised during xmodule recursion,
    """
    for child in item.get_children():
        _xmodule_recurse(child, action, ignore_exception)

    try:
        return action(item)
    except ignore_exception:
        return


def get_parent_xblock(xblock):
    """
    Returns the xblock that is the parent of the specified xblock, or None if it has no parent.
    """
    locator = xblock.location
    parent_locations = modulestore().get_parent_locations(locator,)

    if len(parent_locations) == 0:
        return None
    elif len(parent_locations) > 1:
        logging.error('Multiple parents have been found for %s', unicode(locator))
    return modulestore().get_item(parent_locations[0])


def is_unit(xblock):
    """
    Returns true if the specified xblock is a vertical that is treated as a unit.
    A unit is a vertical that is a direct child of a sequential (aka a subsection).
    """
    if xblock.category == 'vertical':
        parent_xblock = get_parent_xblock(xblock)
        parent_category = parent_xblock.category if parent_xblock else None
        return parent_category == 'sequential'
    return False


def xblock_has_own_studio_page(xblock):
    """
    Returns true if the specified xblock has an associated Studio page. Most xblocks do
    not have their own page but are instead shown on the page of their parent. There
    are a few exceptions:
      1. Courses
      2. Verticals that are either:
        - themselves treated as units (in which case they are shown on a unit page)
        - a direct child of a unit (in which case they are shown on a container page)
      3. XBlocks with children, except for:
        - sequentials (aka subsections)
        - chapters (aka sections)
    """
    category = xblock.category

    if is_unit(xblock):
        return True
    elif category == 'vertical':
        parent_xblock = get_parent_xblock(xblock)
        return is_unit(parent_xblock) if parent_xblock else False
    elif category in ('sequential', 'chapter'):
        return False

    # All other xblocks with children have their own page
    return xblock.has_children


def xblock_studio_url(xblock):
    """
    Returns the Studio editing URL for the specified xblock.
    """
    if not xblock_has_own_studio_page(xblock):
        return None
    category = xblock.category
    parent_xblock = get_parent_xblock(xblock)
    parent_category = parent_xblock.category if parent_xblock else None
    if category == 'course':
        return reverse_course_url('course_handler', xblock.location.course_key)
    elif category == 'vertical' and parent_category == 'sequential':
        # only show the unit page for verticals directly beneath a subsection
        return reverse_usage_url('unit_handler', xblock.location)
    else:
        return reverse_usage_url('container_handler', xblock.location)

########NEW FILE########
__FILENAME__ = import_export
"""
These views handle all actions in Studio related to import and exporting of
courses
"""
import logging
import os
import re
import shutil
import tarfile
from path import path
from tempfile import mkdtemp

from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import SuspiciousOperation, PermissionDenied
from django.core.files.temp import NamedTemporaryFile
from django.core.servers.basehttp import FileWrapper
from django.http import HttpResponse, HttpResponseNotFound
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_http_methods, require_GET

from django_future.csrf import ensure_csrf_cookie
from edxmako.shortcuts import render_to_response
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import SerializationError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_exporter import export_to_xml

from .access import has_course_access

from extract_tar import safetar_extractall
from student import auth
from student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff
from util.json_request import JsonResponse

from contentstore.utils import reverse_course_url, reverse_usage_url


__all__ = ['import_handler', 'import_status_handler', 'export_handler']


log = logging.getLogger(__name__)


# Regex to capture Content-Range header ranges.
CONTENT_RE = re.compile(r"(?P<start>\d{1,11})-(?P<stop>\d{1,11})/(?P<end>\d{1,11})")


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
def import_handler(request, course_key_string):
    """
    The restful handler for importing a course.

    GET
        html: return html page for import page
        json: not supported
    POST or PUT
        json: import a course via the .tar.gz file specified in request.FILES
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()


    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            raise NotImplementedError('coming soon')
        else:
            data_root = path(settings.GITHUB_REPO_ROOT)
            course_subdir = "{0}-{1}-{2}".format(course_key.org, course_key.course, course_key.run)
            course_dir = data_root / course_subdir

            filename = request.FILES['course-data'].name
            if not filename.endswith('.tar.gz'):
                return JsonResponse(
                    {
                        'ErrMsg': _('We only support uploading a .tar.gz file.'),
                        'Stage': 1
                    },
                    status=415
                )
            temp_filepath = course_dir / filename

            if not course_dir.isdir():
                os.mkdir(course_dir)

            logging.debug('importing course to {0}'.format(temp_filepath))

            # Get upload chunks byte ranges
            try:
                matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
                content_range = matches.groupdict()
            except KeyError:    # Single chunk
                # no Content-Range header, so make one that will work
                content_range = {'start': 0, 'stop': 1, 'end': 2}

            # stream out the uploaded files in chunks to disk
            if int(content_range['start']) == 0:
                mode = "wb+"
            else:
                mode = "ab+"
                size = os.path.getsize(temp_filepath)
                # Check to make sure we haven't missed a chunk
                # This shouldn't happen, even if different instances are handling
                # the same session, but it's always better to catch errors earlier.
                if size < int(content_range['start']):
                    log.warning(
                        "Reported range %s does not match size downloaded so far %s",
                        content_range['start'],
                        size
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': _('File upload corrupted. Please try again'),
                            'Stage': 1
                        },
                        status=409
                    )
                # The last request sometimes comes twice. This happens because
                # nginx sends a 499 error code when the response takes too long.
                elif size > int(content_range['stop']) and size == int(content_range['end']):
                    return JsonResponse({'ImportStatus': 1})

            with open(temp_filepath, mode) as temp_file:
                for chunk in request.FILES['course-data'].chunks():
                    temp_file.write(chunk)

            size = os.path.getsize(temp_filepath)

            if int(content_range['stop']) != int(content_range['end']) - 1:
                # More chunks coming
                return JsonResponse({
                    "files": [{
                                  "name": filename,
                                  "size": size,
                                  "deleteUrl": "",
                                  "deleteType": "",
                                  "url": reverse_course_url('import_handler', course_key),
                                  "thumbnailUrl": ""
                              }]
                })

            else:   # This was the last chunk.

                # Use sessions to keep info about import progress
                session_status = request.session.setdefault("import_status", {})
                key = unicode(course_key) + filename
                session_status[key] = 1
                request.session.modified = True

                # Do everything from now on in a try-finally block to make sure
                # everything is properly cleaned up.
                try:

                    tar_file = tarfile.open(temp_filepath)
                    try:
                        safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
                    except SuspiciousOperation as exc:
                        return JsonResponse(
                            {
                                'ErrMsg': 'Unsafe tar file. Aborting import.',
                                'SuspiciousFileOperationMsg': exc.args[0],
                                'Stage': 1
                            },
                            status=400
                        )
                    finally:
                        tar_file.close()

                    session_status[key] = 2
                    request.session.modified = True

                    # find the 'course.xml' file
                    def get_all_files(directory):
                        """
                        For each file in the directory, yield a 2-tuple of (file-name,
                        directory-path)
                        """
                        for dirpath, _dirnames, filenames in os.walk(directory):
                            for filename in filenames:
                                yield (filename, dirpath)

                    def get_dir_for_fname(directory, filename):
                        """
                        Returns the dirpath for the first file found in the directory
                        with the given name.  If there is no file in the directory with
                        the specified name, return None.
                        """
                        for fname, dirpath in get_all_files(directory):
                            if fname == filename:
                                return dirpath
                        return None

                    fname = "course.xml"

                    dirpath = get_dir_for_fname(course_dir, fname)

                    if not dirpath:
                        return JsonResponse(
                            {

                                'ErrMsg': _('Could not find the course.xml file in the package.'),
                                'Stage': 2
                            },
                            status=415
                        )

                    logging.debug('found course.xml at {0}'.format(dirpath))

                    if dirpath != course_dir:
                        for fname in os.listdir(dirpath):
                            shutil.move(dirpath / fname, course_dir)

                    _module_store, course_items = import_from_xml(
                        modulestore('direct'),
                        settings.GITHUB_REPO_ROOT,
                        [course_subdir],
                        load_error_modules=False,
                        static_content_store=contentstore(),
                        target_course_id=course_key,
                        draft_store=modulestore()
                    )

                    new_location = course_items[0].location
                    logging.debug('new course at {0}'.format(new_location))

                    session_status[key] = 3
                    request.session.modified = True

                # Send errors to client with stage at which error occurred.
                except Exception as exception:   # pylint: disable=W0703
                    log.exception(
                        "error importing course"
                    )
                    return JsonResponse(
                        {
                            'ErrMsg': str(exception),
                            'Stage': session_status[key]
                        },
                        status=400
                    )

                finally:
                    shutil.rmtree(course_dir)

                return JsonResponse({'Status': 'OK'})
    elif request.method == 'GET':  # assume html
        course_module = modulestore().get_course(course_key)
        return render_to_response('import.html', {
            'context_course': course_module,
            'successful_import_redirect_url': reverse_course_url('course_handler', course_key),
            'import_status_url': reverse_course_url("import_status_handler", course_key, kwargs={'filename': "fillerName"}),
        })
    else:
        return HttpResponseNotFound()


# pylint: disable=unused-argument
@require_GET
@ensure_csrf_cookie
@login_required
def import_status_handler(request, course_key_string, filename=None):
    """
    Returns an integer corresponding to the status of a file import. These are:

        0 : No status info found (import done or upload still in progress)
        1 : Extracting file
        2 : Validating.
        3 : Importing to mongo

    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    try:
        session_status = request.session["import_status"]
        status = session_status[course_key_string + filename]
    except KeyError:
        status = 0

    return JsonResponse({"ImportStatus": status})


# pylint: disable=unused-argument
@ensure_csrf_cookie
@login_required
@require_http_methods(("GET",))
def export_handler(request, course_key_string):
    """
    The restful handler for exporting a course.

    GET
        html: return html page for import page
        application/x-tgz: return tar.gz file containing exported course
        json: not supported

    Note that there are 2 ways to request the tar.gz file. The request header can specify
    application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).

    If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
    which describes the error.
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)

    # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
    requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))

    export_url = reverse_course_url('export_handler', course_key) + '?_accept=application/x-tgz'
    if 'application/x-tgz' in requested_format:
        name = course_module.url_name
        export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
        root_dir = path(mkdtemp())

        try:
            export_to_xml(modulestore('direct'), contentstore(), course_module.id, root_dir, name, modulestore())

            logging.debug('tar file being generated at {0}'.format(export_file.name))
            with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
                tar_file.add(root_dir / name, arcname=name)
        except SerializationError as exc:
            log.exception('There was an error exporting course %s', course_module.id)
            unit = None
            failed_item = None
            parent = None
            try:
                failed_item = modulestore().get_item(exc.location)
                parent_locs = modulestore().get_parent_locations(failed_item.location)

                if len(parent_locs) > 0:
                    parent = modulestore().get_item(parent_locs[0])
                    if parent.location.category == 'vertical':
                        unit = parent
            except:  # pylint: disable=bare-except
                # if we have a nested exception, then we'll show the more generic error message
                pass

            return render_to_response('export.html', {
                'context_course': course_module,
                'in_err': True,
                'raw_err_msg': str(exc),
                'failed_module': failed_item,
                'unit': unit,
                'edit_unit_url': reverse_usage_url("unit_handler", parent.location) if parent else "",
                'course_home_url': reverse_course_url("course_handler", course_key),
                'export_url': export_url
            })
        except Exception as exc:
            log.exception('There was an error exporting course %s', course_module.id)
            return render_to_response('export.html', {
                'context_course': course_module,
                'in_err': True,
                'unit': None,
                'raw_err_msg': str(exc),
                'course_home_url': reverse_course_url("course_handler", course_key),
                'export_url': export_url
            })
        finally:
            shutil.rmtree(root_dir / name)

        wrapper = FileWrapper(export_file)
        response = HttpResponse(wrapper, content_type='application/x-tgz')
        response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
        response['Content-Length'] = os.path.getsize(export_file.name)
        return response

    elif 'text/html' in requested_format:
        return render_to_response('export.html', {
            'context_course': course_module,
            'export_url': export_url
        })

    else:
        # Only HTML or x-tgz request formats are supported (no JSON).
        return HttpResponse(status=406)

########NEW FILE########
__FILENAME__ = item
"""Views for items (modules)."""
from __future__ import absolute_import

import hashlib
import logging
from uuid import uuid4

from collections import OrderedDict
from functools import partial
from static_replace import replace_static_urls
from xmodule_modifiers import wrap_xblock

from django.core.exceptions import PermissionDenied
from django.contrib.auth.decorators import login_required
from django.http import HttpResponseBadRequest, HttpResponse, Http404
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_http_methods

from xblock.fields import Scope
from xblock.fragment import Fragment

import xmodule
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError, DuplicateItemError
from xmodule.modulestore.inheritance import own_metadata
from xmodule.video_module import manage_video_subtitles_save

from util.json_request import expect_json, JsonResponse
from util.string_utils import str_to_bool

from ..utils import get_modulestore

from .access import has_course_access
from .helpers import _xmodule_recurse, xblock_has_own_studio_page
from contentstore.utils import compute_publish_state, PublishState
from xmodule.modulestore.draft import DIRECT_ONLY_CATEGORIES
from contentstore.views.preview import get_preview_fragment
from edxmako.shortcuts import render_to_string
from models.settings.course_grading import CourseGradingModel
from cms.lib.xblock.runtime import handler_url, local_resource_url
from xmodule.modulestore.keys import UsageKey, CourseKey

__all__ = ['orphan_handler', 'xblock_handler', 'xblock_view_handler']

log = logging.getLogger(__name__)

CREATE_IF_NOT_FOUND = ['course_info']


# In order to allow descriptors to use a handler url, we need to
# monkey-patch the x_module library.
# TODO: Remove this code when Runtimes are no longer created by modulestores
xmodule.x_module.descriptor_global_handler_url = handler_url
xmodule.x_module.descriptor_global_local_resource_url = local_resource_url


def hash_resource(resource):
    """
    Hash a :class:`xblock.fragment.FragmentResource`.
    """
    md5 = hashlib.md5()
    md5.update(repr(resource))
    return md5.hexdigest()


# pylint: disable=unused-argument
@require_http_methods(("DELETE", "GET", "PUT", "POST"))
@login_required
@expect_json
def xblock_handler(request, usage_key_string):
    """
    The restful handler for xblock requests.

    DELETE
        json: delete this xblock instance from the course. Supports query parameters "recurse" to delete
        all children and "all_versions" to delete from all (mongo) versions.
    GET
        json: returns representation of the xblock (locator id, data, and metadata).
              if ?fields=graderType, it returns the graderType for the unit instead of the above.
        html: returns HTML for rendering the xblock (which includes both the "preview" view and the "editor" view)
    PUT or POST
        json: if xblock locator is specified, update the xblock instance. The json payload can contain
              these fields, all optional:
                :data: the new value for the data.
                :children: the unicode representation of the UsageKeys of children for this xblock.
                :metadata: new values for the metadata fields. Any whose values are None will be deleted not set
                       to None! Absent ones will be left alone.
                :nullout: which metadata fields to set to None
                :graderType: change how this unit is graded
                :publish: can be one of three values, 'make_public, 'make_private', or 'create_draft'
              The JSON representation on the updated xblock (minus children) is returned.

              if usage_key_string is not specified, create a new xblock instance, either by duplicating
              an existing xblock, or creating an entirely new one. The json playload can contain
              these fields:
                :parent_locator: parent for new xblock, required for both duplicate and create new instance
                :duplicate_source_locator: if present, use this as the source for creating a duplicate copy
                :category: type of xblock, required if duplicate_source_locator is not present.
                :display_name: name for new xblock, optional
                :boilerplate: template name for populating fields, optional and only used
                     if duplicate_source_locator is not present
              The locator (unicode representation of a UsageKey) for the created xblock (minus children) is returned.
    """
    if usage_key_string:
        usage_key = UsageKey.from_string(usage_key_string)
        if not has_course_access(request.user, usage_key.course_key):
            raise PermissionDenied()

        if request.method == 'GET':
            accept_header = request.META.get('HTTP_ACCEPT', 'application/json')

            if 'application/json' in accept_header:
                fields = request.REQUEST.get('fields', '').split(',')
                if 'graderType' in fields:
                    # right now can't combine output of this w/ output of _get_module_info, but worthy goal
                    return JsonResponse(CourseGradingModel.get_section_grader_type(usage_key))
                # TODO: pass fields to _get_module_info and only return those
                rsp = _get_module_info(usage_key)
                return JsonResponse(rsp)
            else:
                return HttpResponse(status=406)

        elif request.method == 'DELETE':
            delete_children = str_to_bool(request.REQUEST.get('recurse', 'False'))
            delete_all_versions = str_to_bool(request.REQUEST.get('all_versions', 'False'))

            return _delete_item_at_location(usage_key, delete_children, delete_all_versions, request.user)
        else:  # Since we have a usage_key, we are updating an existing xblock.
            return _save_item(
                request,
                usage_key,
                data=request.json.get('data'),
                children=request.json.get('children'),
                metadata=request.json.get('metadata'),
                nullout=request.json.get('nullout'),
                grader_type=request.json.get('graderType'),
                publish=request.json.get('publish'),
            )
    elif request.method in ('PUT', 'POST'):
        if 'duplicate_source_locator' in request.json:
            parent_usage_key = UsageKey.from_string(request.json['parent_locator'])
            duplicate_source_usage_key = UsageKey.from_string(request.json['duplicate_source_locator'])

            dest_usage_key = _duplicate_item(
                parent_usage_key,
                duplicate_source_usage_key,
                request.json.get('display_name'),
                request.user,
            )

            return JsonResponse({"locator": unicode(dest_usage_key)})
        else:
            return _create_item(request)
    else:
        return HttpResponseBadRequest(
            "Only instance creation is supported without a usage key.",
            content_type="text/plain"
        )

# pylint: disable=unused-argument
@require_http_methods(("GET"))
@login_required
@expect_json
def xblock_view_handler(request, usage_key_string, view_name):
    """
    The restful handler for requests for rendered xblock views.

    Returns a json object containing two keys:
        html: The rendered html of the view
        resources: A list of tuples where the first element is the resource hash, and
            the second is the resource description
    """
    usage_key = UsageKey.from_string(usage_key_string)
    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    accept_header = request.META.get('HTTP_ACCEPT', 'application/json')

    if 'application/json' in accept_header:
        store = get_modulestore(usage_key)
        xblock = store.get_item(usage_key)
        is_read_only = _is_xblock_read_only(xblock)
        container_views = ['container_preview', 'reorderable_container_child_preview']
        unit_views = ['student_view']

        # wrap the generated fragment in the xmodule_editor div so that the javascript
        # can bind to it correctly
        xblock.runtime.wrappers.append(partial(wrap_xblock, 'StudioRuntime', usage_id_serializer=unicode))

        if view_name == 'studio_view':
            try:
                fragment = xblock.render('studio_view')
            # catch exceptions indiscriminately, since after this point they escape the
            # dungeon and surface as uneditable, unsaveable, and undeletable
            # component-goblins.
            except Exception as exc:                          # pylint: disable=w0703
                log.debug("unable to render studio_view for %r", xblock, exc_info=True)
                fragment = Fragment(render_to_string('html_error.html', {'message': str(exc)}))

            # change not authored by requestor but by xblocks.
            store.update_item(xblock, None)

        elif view_name == 'student_view' and xblock_has_own_studio_page(xblock):
            context = {
                'runtime_type': 'studio',
                'container_view': False,
                'read_only': is_read_only,
                'root_xblock': xblock,
            }
            # For non-leaf xblocks on the unit page, show the special rendering
            # which links to the new container page.
            html = render_to_string('container_xblock_component.html', {
                'xblock_context': context,
                'xblock': xblock,
                'locator': usage_key,
            })
            return JsonResponse({
                'html': html,
                'resources': [],
            })
        elif view_name in (unit_views + container_views):
            is_container_view = (view_name in container_views)

            # Determine the items to be shown as reorderable. Note that the view
            # 'reorderable_container_child_preview' is only rendered for xblocks that
            # are being shown in a reorderable container, so the xblock is automatically
            # added to the list.
            reorderable_items = set()
            if view_name == 'reorderable_container_child_preview':
                reorderable_items.add(xblock.location)

            # Only show the new style HTML for the container view, i.e. for non-verticals
            # Note: this special case logic can be removed once the unit page is replaced
            # with the new container view.
            context = {
                'runtime_type': 'studio',
                'container_view': is_container_view,
                'read_only': is_read_only,
                'root_xblock': xblock if (view_name == 'container_preview') else None,
                'reorderable_items': reorderable_items
            }

            fragment = get_preview_fragment(request, xblock, context)
            # For old-style pages (such as unit and static pages), wrap the preview with
            # the component div. Note that the container view recursively adds headers
            # into the preview fragment, so we don't want to add another header here.
            if not is_container_view:
                fragment.content = render_to_string('component.html', {
                    'xblock_context': context,
                    'preview': fragment.content,
                    'label': xblock.display_name or xblock.scope_ids.block_type,
                })
        else:
            raise Http404

        hashed_resources = OrderedDict()
        for resource in fragment.resources:
            hashed_resources[hash_resource(resource)] = resource

        return JsonResponse({
            'html': fragment.content,
            'resources': hashed_resources.items()
        })

    else:
        return HttpResponse(status=406)


def _is_xblock_read_only(xblock):
    """
    Returns true if the specified xblock is read-only, meaning that it cannot be edited.
    """
    # We allow direct editing of xblocks in DIRECT_ONLY_CATEGORIES (for example, static pages).
    if xblock.category in DIRECT_ONLY_CATEGORIES:
        return False
    component_publish_state = compute_publish_state(xblock)
    return component_publish_state == PublishState.public


def _save_item(request, usage_key, data=None, children=None, metadata=None, nullout=None,
               grader_type=None, publish=None):
    """
    Saves xblock w/ its fields. Has special processing for grader_type, publish, and nullout and Nones in metadata.
    nullout means to truly set the field to None whereas nones in metadata mean to unset them (so they revert
    to default).
    """
    store = get_modulestore(usage_key)

    try:
        existing_item = store.get_item(usage_key)
    except ItemNotFoundError:
        if usage_key.category in CREATE_IF_NOT_FOUND:
            # New module at this location, for pages that are not pre-created.
            # Used for course info handouts.
            store.create_and_save_xmodule(usage_key)
            existing_item = store.get_item(usage_key)
        else:
            raise
    except InvalidLocationError:
        log.error("Can't find item by location.")
        return JsonResponse({"error": "Can't find item by location: " + unicode(usage_key)}, 404)

    old_metadata = own_metadata(existing_item)

    if publish:
        if publish == 'make_private':
            _xmodule_recurse(
                existing_item,
                lambda i: modulestore().unpublish(i.location),
                ignore_exception=ItemNotFoundError
            )
        elif publish == 'create_draft':
            # This recursively clones the existing item location to a draft location (the draft is
            # implicit, because modulestore is a Draft modulestore)
            _xmodule_recurse(
                existing_item,
                lambda i: modulestore().convert_to_draft(i.location),
                ignore_exception=DuplicateItemError
            )

    if data:
        # TODO Allow any scope.content fields not just "data" (exactly like the get below this)
        existing_item.data = data
    else:
        data = existing_item.get_explicitly_set_fields_by_scope(Scope.content)

    if children is not None:
        children_usage_keys = [
            UsageKey.from_string(child)
            for child
            in children
        ]
        existing_item.children = children_usage_keys

    # also commit any metadata which might have been passed along
    if nullout is not None or metadata is not None:
        # the postback is not the complete metadata, as there's system metadata which is
        # not presented to the end-user for editing. So let's use the original (existing_item) and
        # 'apply' the submitted metadata, so we don't end up deleting system metadata.
        if nullout is not None:
            for metadata_key in nullout:
                setattr(existing_item, metadata_key, None)

        # update existing metadata with submitted metadata (which can be partial)
        # IMPORTANT NOTE: if the client passed 'null' (None) for a piece of metadata that means 'remove it'. If
        # the intent is to make it None, use the nullout field
        if metadata is not None:
            for metadata_key, value in metadata.items():
                field = existing_item.fields[metadata_key]

                if value is None:
                    field.delete_from(existing_item)
                else:
                    try:
                        value = field.from_json(value)
                    except ValueError:
                        return JsonResponse({"error": "Invalid data"}, 400)
                    field.write_to(existing_item, value)

        if existing_item.category == 'video':
            manage_video_subtitles_save(existing_item, request.user, old_metadata, generate_translation=True)

    # commit to datastore
    store.update_item(existing_item, request.user.id)

    result = {
        'id': unicode(usage_key),
        'data': data,
        'metadata': own_metadata(existing_item)
    }

    if grader_type is not None:
        result.update(CourseGradingModel.update_section_grader_type(existing_item, grader_type, request.user))

    # Make public after updating the xblock, in case the caller asked
    # for both an update and a publish.
    if publish and publish == 'make_public':
        def _publish(block):
            # This is super gross, but prevents us from publishing something that
            # we shouldn't. Ideally, all modulestores would have a consistant
            # interface for publishing. However, as of now, only the DraftMongoModulestore
            # does, so we have to check for the attribute explicitly.
            store = get_modulestore(block.location)
            store.publish(block.location, request.user.id)

        _xmodule_recurse(
            existing_item,
            _publish
        )

    # Note that children aren't being returned until we have a use case.
    return JsonResponse(result)


@login_required
@expect_json
def _create_item(request):
    """View for create items."""
    usage_key = UsageKey.from_string(request.json['parent_locator'])
    category = request.json['category']

    display_name = request.json.get('display_name')

    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    parent = get_modulestore(category).get_item(usage_key)
    dest_usage_key = usage_key.replace(category=category, name=uuid4().hex)

    # get the metadata, display_name, and definition from the request
    metadata = {}
    data = None
    template_id = request.json.get('boilerplate')
    if template_id:
        clz = parent.runtime.load_block_type(category)
        if clz is not None:
            template = clz.get_template(template_id)
            if template is not None:
                metadata = template.get('metadata', {})
                data = template.get('data')

    if display_name is not None:
        metadata['display_name'] = display_name

    get_modulestore(category).create_and_save_xmodule(
        dest_usage_key,
        definition_data=data,
        metadata=metadata,
        system=parent.runtime,
    )

    # TODO replace w/ nicer accessor
    if not 'detached' in parent.runtime.load_block_type(category)._class_tags:
        parent.children.append(dest_usage_key)
        get_modulestore(parent.location).update_item(parent, request.user.id)

    return JsonResponse({"locator": unicode(dest_usage_key), "courseKey": unicode(dest_usage_key.course_key)})


def _duplicate_item(parent_usage_key, duplicate_source_usage_key, display_name=None, user=None):
    """
    Duplicate an existing xblock as a child of the supplied parent_usage_key.
    """
    store = get_modulestore(duplicate_source_usage_key)
    source_item = store.get_item(duplicate_source_usage_key)
    # Change the blockID to be unique.
    dest_usage_key = duplicate_source_usage_key.replace(name=uuid4().hex)
    category = dest_usage_key.category

    # Update the display name to indicate this is a duplicate (unless display name provided).
    duplicate_metadata = own_metadata(source_item)
    if display_name is not None:
        duplicate_metadata['display_name'] = display_name
    else:
        if source_item.display_name is None:
            duplicate_metadata['display_name'] = _("Duplicate of {0}").format(source_item.category)
        else:
            duplicate_metadata['display_name'] = _("Duplicate of '{0}'").format(source_item.display_name)

    get_modulestore(category).create_and_save_xmodule(
        dest_usage_key,
        definition_data=source_item.data if hasattr(source_item, 'data') else None,
        metadata=duplicate_metadata,
        system=source_item.runtime,
    )

    dest_module = get_modulestore(category).get_item(dest_usage_key)
    # Children are not automatically copied over (and not all xblocks have a 'children' attribute).
    # Because DAGs are not fully supported, we need to actually duplicate each child as well.
    if source_item.has_children:
        dest_module.children = []
        for child in source_item.children:
            dupe = _duplicate_item(dest_usage_key, child, user=user)
            dest_module.children.append(dupe)
        get_modulestore(dest_usage_key).update_item(dest_module, user.id if user else None)

    if not 'detached' in source_item.runtime.load_block_type(category)._class_tags:
        parent = get_modulestore(parent_usage_key).get_item(parent_usage_key)
        # If source was already a child of the parent, add duplicate immediately afterward.
        # Otherwise, add child to end.
        if duplicate_source_usage_key in parent.children:
            source_index = parent.children.index(duplicate_source_usage_key)
            parent.children.insert(source_index + 1, dest_usage_key)
        else:
            parent.children.append(dest_usage_key)
        get_modulestore(parent_usage_key).update_item(parent, user.id if user else None)

    return dest_usage_key


def _delete_item_at_location(item_usage_key, delete_children=False, delete_all_versions=False, user=None):
    """
    Deletes the item at with the given Location.

    It is assumed that course permissions have already been checked.
    """
    store = get_modulestore(item_usage_key)

    item = store.get_item(item_usage_key)

    if delete_children:
        _xmodule_recurse(item, lambda i: store.delete_item(i.location, delete_all_versions=delete_all_versions))
    else:
        store.delete_item(item.location, delete_all_versions=delete_all_versions)

    # cdodge: we need to remove our parent's pointer to us so that it is no longer dangling
    if delete_all_versions:
        parent_locs = modulestore('direct').get_parent_locations(item_usage_key)

        for parent_loc in parent_locs:
            parent = modulestore('direct').get_item(parent_loc)
            parent.children.remove(item_usage_key)
            modulestore('direct').update_item(parent, user.id if user else None)

    return JsonResponse()


# pylint: disable=W0613
@login_required
@require_http_methods(("GET", "DELETE"))
def orphan_handler(request, course_key_string):
    """
    View for handling orphan related requests. GET gets all of the current orphans.
    DELETE removes all orphans (requires is_staff access)

    An orphan is a block whose category is not in the DETACHED_CATEGORY list, is not the root, and is not reachable
    from the root via children
    """
    course_usage_key = CourseKey.from_string(course_key_string)
    if request.method == 'GET':
        if has_course_access(request.user, course_usage_key):
            return JsonResponse(modulestore().get_orphans(course_usage_key))
        else:
            raise PermissionDenied()
    if request.method == 'DELETE':
        if request.user.is_staff:
            items = modulestore().get_orphans(course_usage_key)
            for itemloc in items:
                # get_orphans returns the deprecated string format
                usage_key = course_usage_key.make_usage_key_from_deprecated_string(itemloc)
                modulestore('draft').delete_item(usage_key, delete_all_versions=True)
            return JsonResponse({'deleted': items})
        else:
            raise PermissionDenied()


def _get_module_info(usage_key, rewrite_static_links=True):
    """
    metadata, data, id representation of a leaf module fetcher.
    :param usage_key: A UsageKey
    """
    store = get_modulestore(usage_key)
    try:
        module = store.get_item(usage_key)
    except ItemNotFoundError:
        if usage_key.category in CREATE_IF_NOT_FOUND:
            # Create a new one for certain categories only. Used for course info handouts.
            store.create_and_save_xmodule(usage_key)
            module = store.get_item(usage_key)
        else:
            raise

    data = getattr(module, 'data', '')
    if rewrite_static_links:
        data = replace_static_urls(
            data,
            None,
            course_id=usage_key.course_key
        )

    # Note that children aren't being returned until we have a use case.
    return {
        'id': unicode(usage_key),
        'data': data,
        'metadata': own_metadata(module)
    }

########NEW FILE########
__FILENAME__ = preview
from __future__ import absolute_import

import logging
from functools import partial

from django.conf import settings
from django.core.urlresolvers import reverse
from django.http import Http404, HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from edxmako.shortcuts import render_to_string

from xmodule_modifiers import replace_static_urls, wrap_xblock, wrap_fragment
from xmodule.error_module import ErrorDescriptor
from xmodule.exceptions import NotFoundError, ProcessingError
from xmodule.modulestore.django import modulestore, ModuleI18nService
from xmodule.modulestore.keys import UsageKey
from xmodule.x_module import ModuleSystem
from xblock.runtime import KvsFieldData
from xblock.django.request import webob_to_django_response, django_to_webob_request
from xblock.exceptions import NoSuchHandlerError
from xblock.fragment import Fragment

from lms.lib.xblock.field_data import LmsFieldData
from cms.lib.xblock.runtime import local_resource_url

from util.sandboxing import can_execute_unsafe_code

import static_replace
from .session_kv_store import SessionKeyValueStore
from .helpers import render_from_lms, xblock_has_own_studio_page

from contentstore.views.access import get_user_role

__all__ = ['preview_handler']

log = logging.getLogger(__name__)


@login_required
def preview_handler(request, usage_key_string, handler, suffix=''):
    """
    Dispatch an AJAX action to an xblock

    usage_key_string: The usage_key_string-id of the block to dispatch to, passed through `quote_slashes`
    handler: The handler to execute
    suffix: The remainder of the url to be passed to the handler
    """
    usage_key = UsageKey.from_string(usage_key_string)

    descriptor = modulestore().get_item(usage_key)
    instance = _load_preview_module(request, descriptor)
    # Let the module handle the AJAX
    req = django_to_webob_request(request)
    try:
        resp = instance.handle(handler, req, suffix)

    except NoSuchHandlerError:
        log.exception("XBlock %s attempted to access missing handler %r", instance, handler)
        raise Http404

    except NotFoundError:
        log.exception("Module indicating to user that request doesn't exist")
        raise Http404

    except ProcessingError:
        log.warning("Module raised an error while processing AJAX request",
                    exc_info=True)
        return HttpResponseBadRequest()

    except Exception:
        log.exception("error processing ajax call")
        raise

    return webob_to_django_response(resp)


class PreviewModuleSystem(ModuleSystem):  # pylint: disable=abstract-method
    """
    An XModule ModuleSystem for use in Studio previews
    """
    # xmodules can check for this attribute during rendering to determine if
    # they are being rendered for preview (i.e. in Studio)
    is_author_mode = True

    def handler_url(self, block, handler_name, suffix='', query='', thirdparty=False):
        return reverse('preview_handler', kwargs={
            'usage_key_string': unicode(block.location),
            'handler': handler_name,
            'suffix': suffix,
        }) + '?' + query

    def local_resource_url(self, block, uri):
        return local_resource_url(block, uri)


def _preview_module_system(request, descriptor):
    """
    Returns a ModuleSystem for the specified descriptor that is specialized for
    rendering module previews.

    request: The active django request
    descriptor: An XModuleDescriptor
    """

    course_id = descriptor.location.course_key
    display_name_only = (descriptor.category == 'static_tab')

    wrappers = [
        # This wrapper wraps the module in the template specified above
        partial(wrap_xblock, 'PreviewRuntime', display_name_only=display_name_only, usage_id_serializer=unicode),

        # This wrapper replaces urls in the output that start with /static
        # with the correct course-specific url for the static content
        partial(replace_static_urls, None, course_id=course_id),
        _studio_wrap_xblock,
    ]

    return PreviewModuleSystem(
        static_url=settings.STATIC_URL,
        # TODO (cpennington): Do we want to track how instructors are using the preview problems?
        track_function=lambda event_type, event: None,
        filestore=descriptor.runtime.resources_fs,
        get_module=partial(_load_preview_module, request),
        render_template=render_from_lms,
        debug=True,
        replace_urls=partial(static_replace.replace_static_urls, data_directory=None, course_id=course_id),
        user=request.user,
        can_execute_unsafe_code=(lambda: can_execute_unsafe_code(course_id)),
        mixins=settings.XBLOCK_MIXINS,
        course_id=course_id,
        anonymous_student_id='student',

        # Set up functions to modify the fragment produced by student_view
        wrappers=wrappers,
        error_descriptor_class=ErrorDescriptor,
        get_user_role=lambda: get_user_role(request.user, course_id),
        descriptor_runtime=descriptor.runtime,
        services={
            "i18n": ModuleI18nService(),
        },
    )


def _load_preview_module(request, descriptor):
    """
    Return a preview XModule instantiated from the supplied descriptor.

    request: The active django request
    descriptor: An XModuleDescriptor
    """
    student_data = KvsFieldData(SessionKeyValueStore(request))
    descriptor.bind_for_student(
        _preview_module_system(request, descriptor),
        LmsFieldData(descriptor._field_data, student_data),  # pylint: disable=protected-access
    )
    return descriptor


def _is_xblock_reorderable(xblock, context):
    """
    Returns true if the specified xblock is in the set of reorderable xblocks.
    """
    return xblock.location in context['reorderable_items']


# pylint: disable=unused-argument
def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False):
    """
    Wraps the results of rendering an XBlock view in a div which adds a header and Studio action buttons.
    """
    # Only add the Studio wrapper when on the container page. The unit page will remain as is for now.
    if context.get('container_view', None) and view == 'student_view':
        root_xblock = context.get('root_xblock')
        is_root = root_xblock and xblock.location == root_xblock.location
        is_reorderable = _is_xblock_reorderable(xblock, context)
        template_context = {
            'xblock_context': context,
            'xblock': xblock,
            'content': frag.content,
            'is_root': is_root,
            'is_reorderable': is_reorderable,
        }
        # For child xblocks with their own page, render a link to the page
        if xblock_has_own_studio_page(xblock) and not is_root:
            template = 'studio_container_wrapper.html'
        else:
            template = 'studio_xblock_wrapper.html'
        html = render_to_string(template, template_context)
        frag = wrap_fragment(frag, html)
    return frag


def get_preview_fragment(request, descriptor, context):
    """
    Returns the HTML returned by the XModule's student_view,
    specified by the descriptor and idx.
    """
    module = _load_preview_module(request, descriptor)

    try:
        fragment = module.render("student_view", context)
    except Exception as exc:                          # pylint: disable=W0703
        log.warning("Unable to render student_view for %r", module, exc_info=True)
        fragment = Fragment(render_to_string('html_error.html', {'message': str(exc)}))
    return fragment

########NEW FILE########
__FILENAME__ = public
"""
Public views
"""
from django_future.csrf import ensure_csrf_cookie
from django.core.context_processors import csrf
from django.core.urlresolvers import reverse
from django.shortcuts import redirect
from django.conf import settings

from edxmako.shortcuts import render_to_response

from external_auth.views import (ssl_login_shortcut, ssl_get_cert_from_request,
                                 redirect_with_get)
from microsite_configuration import microsite

__all__ = ['signup', 'login_page', 'howitworks']


@ensure_csrf_cookie
def signup(request):
    """
    Display the signup form.
    """
    csrf_token = csrf(request)['csrf_token']
    if request.user.is_authenticated():
        return redirect('/course/')
    if settings.FEATURES.get('AUTH_USE_CERTIFICATES_IMMEDIATE_SIGNUP'):
        # Redirect to course to login to process their certificate if SSL is enabled
        # and registration is disabled.
        return redirect_with_get('login', request.GET, False)

    return render_to_response('register.html', {'csrf': csrf_token})


@ssl_login_shortcut
@ensure_csrf_cookie
def login_page(request):
    """
    Display the login form.
    """
    csrf_token = csrf(request)['csrf_token']
    if (settings.FEATURES['AUTH_USE_CERTIFICATES'] and
            ssl_get_cert_from_request(request)):
        # SSL login doesn't require a login view, so redirect
        # to course now that the user is authenticated via
        # the decorator.
        next_url = request.GET.get('next')
        if next_url:
            return redirect(next_url)
        else:
            return redirect('/course/')
    if settings.FEATURES.get('AUTH_USE_CAS'):
        # If CAS is enabled, redirect auth handling to there
        return redirect(reverse('cas-login'))

    return render_to_response(
        'login.html',
        {
            'csrf': csrf_token,
            'forgot_password_link': "//{base}/login#forgot-password-modal".format(base=settings.LMS_BASE),
            'platform_name': microsite.get_value('platform_name', settings.PLATFORM_NAME),
        }
    )


def howitworks(request):
    "Proxy view"
    if request.user.is_authenticated():
        return redirect('/course/')
    else:
        return render_to_response('howitworks.html', {})

########NEW FILE########
__FILENAME__ = session_kv_store
"""
An :class:`~xblock.runtime.KeyValueStore` that stores data in the django session
"""
from __future__ import absolute_import

from xblock.runtime import KeyValueStore

class SessionKeyValueStore(KeyValueStore):
    def __init__(self, request):
        self._session = request.session

    def get(self, key):
        return self._session[tuple(key)]

    def set(self, key, value):
        self._session[tuple(key)] = value

    def delete(self, key):
        del self._session[tuple(key)]

    def has(self, key):
        return tuple(key) in self._session

########NEW FILE########
__FILENAME__ = tabs
"""
Views related to course tabs
"""
from access import has_course_access
from util.json_request import expect_json, JsonResponse

from django.http import HttpResponseNotFound
from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django_future.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_http_methods
from edxmako.shortcuts import render_to_response
from xmodule.modulestore.django import modulestore
from xmodule.tabs import CourseTabList, StaticTab, CourseTab, InvalidTabsException
from xmodule.modulestore.keys import CourseKey, UsageKey

from ..utils import get_lms_link_for_item

__all__ = ['tabs_handler']

@expect_json
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT"))
def tabs_handler(request, course_key_string):
    """
    The restful handler for static tabs.

    GET
        html: return page for editing static tabs
        json: not supported
    PUT or POST
        json: update the tab order. It is expected that the request body contains a JSON-encoded dict with entry "tabs".
        The value for "tabs" is an array of tab locators, indicating the desired order of the tabs.

    Creating a tab, deleting a tab, or changing its contents is not supported through this method.
    Instead use the general xblock URL (see item.xblock_handler).
    """
    course_key = CourseKey.from_string(course_key_string)
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_item = modulestore().get_course(course_key)

    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        if request.method == 'GET':
            raise NotImplementedError('coming soon')
        else:
            if 'tabs' in request.json:
                return reorder_tabs_handler(course_item, request)
            elif 'tab_id_locator' in request.json:
                return edit_tab_handler(course_item, request)
            else:
                raise NotImplementedError('Creating or changing tab content is not supported.')

    elif request.method == 'GET':  # assume html
        # get all tabs from the tabs list: static tabs (a.k.a. user-created tabs) and built-in tabs
        # present in the same order they are displayed in LMS

        tabs_to_render = []
        for tab in CourseTabList.iterate_displayable_cms(
                course_item,
                settings,
        ):
            if isinstance(tab, StaticTab):
                # static tab needs its locator information to render itself as an xmodule
                static_tab_loc = course_key.make_usage_key('static_tab', tab.url_slug)
                tab.locator = static_tab_loc
            tabs_to_render.append(tab)

        return render_to_response('edit-tabs.html', {
            'context_course': course_item,
            'tabs_to_render': tabs_to_render,
            'lms_link': get_lms_link_for_item(course_item.location),
        })
    else:
        return HttpResponseNotFound()


def reorder_tabs_handler(course_item, request):
    """
    Helper function for handling reorder of tabs request
    """

    # Tabs are identified by tab_id or locators.
    # The locators are used to identify static tabs since they are xmodules.
    # Although all tabs have tab_ids, newly created static tabs do not know
    # their tab_ids since the xmodule editor uses only locators to identify new objects.
    requested_tab_id_locators = request.json['tabs']

    # original tab list in original order
    old_tab_list = course_item.tabs

    # create a new list in the new order
    new_tab_list = []
    for tab_id_locator in requested_tab_id_locators:
        tab = get_tab_by_tab_id_locator(old_tab_list, tab_id_locator)
        if tab is None:
            return JsonResponse(
                {"error": "Tab with id_locator '{0}' does not exist.".format(tab_id_locator)}, status=400
            )
        new_tab_list.append(tab)

    # the old_tab_list may contain additional tabs that were not rendered in the UI because of
    # global or course settings.  so add those to the end of the list.
    non_displayed_tabs = set(old_tab_list) - set(new_tab_list)
    new_tab_list.extend(non_displayed_tabs)

    # validate the tabs to make sure everything is Ok (e.g., did the client try to reorder unmovable tabs?)
    try:
        CourseTabList.validate_tabs(new_tab_list)
    except InvalidTabsException, exception:
        return JsonResponse(
            {"error": "New list of tabs is not valid: {0}.".format(str(exception))}, status=400
        )

    # persist the new order of the tabs
    course_item.tabs = new_tab_list
    modulestore('direct').update_item(course_item, request.user.id)

    return JsonResponse()


def edit_tab_handler(course_item, request):
    """
    Helper function for handling requests to edit settings of a single tab
    """

    # Tabs are identified by tab_id or locator
    tab_id_locator = request.json['tab_id_locator']

    # Find the given tab in the course
    tab = get_tab_by_tab_id_locator(course_item.tabs, tab_id_locator)
    if tab is None:
        return JsonResponse(
            {"error": "Tab with id_locator '{0}' does not exist.".format(tab_id_locator)}, status=400
        )

    if 'is_hidden' in request.json:
        # set the is_hidden attribute on the requested tab
        tab.is_hidden = request.json['is_hidden']
        modulestore('direct').update_item(course_item, request.user.id)
    else:
        raise NotImplementedError('Unsupported request to edit tab: {0}'.format(request.json))

    return JsonResponse()


def get_tab_by_tab_id_locator(tab_list, tab_id_locator):
    """
    Look for a tab with the specified tab_id or locator.  Returns the first matching tab.
    """
    if 'tab_id' in tab_id_locator:
        tab = CourseTabList.get_tab_by_id(tab_list, tab_id_locator['tab_id'])
    elif 'tab_locator' in tab_id_locator:
        tab = get_tab_by_locator(tab_list, tab_id_locator['tab_locator'])
    return tab


def get_tab_by_locator(tab_list, usage_key_string):
    """
    Look for a tab with the specified locator.  Returns the first matching tab.
    """
    tab_location = UsageKey.from_string(usage_key_string)
    item = modulestore('direct').get_item(tab_location)
    static_tab = StaticTab(
        name=item.display_name,
        url_slug=item.location.name,
    )
    return CourseTabList.get_tab_by_id(tab_list, static_tab.tab_id)


# "primitive" tab edit functions driven by the command line.
# These should be replaced/deleted by a more capable GUI someday.
# Note that the command line UI identifies the tabs with 1-based
# indexing, but this implementation code is standard 0-based.

def validate_args(num, tab_type):
    "Throws for the disallowed cases."
    if num <= 1:
        raise ValueError('Tabs 1 and 2 cannot be edited')
    if tab_type == 'static_tab':
        raise ValueError('Tabs of type static_tab cannot be edited here (use Studio)')


def primitive_delete(course, num):
    "Deletes the given tab number (0 based)."
    tabs = course.tabs
    validate_args(num, tabs[num].get('type', ''))
    del tabs[num]
    # Note for future implementations: if you delete a static_tab, then Chris Dodge
    # points out that there's other stuff to delete beyond this element.
    # This code happens to not delete static_tab so it doesn't come up.
    modulestore('direct').update_item(course, '**replace_user**')


def primitive_insert(course, num, tab_type, name):
    "Inserts a new tab at the given number (0 based)."
    validate_args(num, tab_type)
    new_tab = CourseTab.from_json({u'type': unicode(tab_type), u'name': unicode(name)})
    tabs = course.tabs
    tabs.insert(num, new_tab)
    modulestore('direct').update_item(course, '**replace_user**')


########NEW FILE########
__FILENAME__ = test_access
"""
Tests access.py
"""
from django.test import TestCase
from django.contrib.auth.models import User

from student.roles import CourseInstructorRole, CourseStaffRole
from student.tests.factories import AdminFactory
from student.auth import add_users
from contentstore.views.access import get_user_role
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class RolesTest(TestCase):
    """
    Tests for lti user role serialization.
    """
    def setUp(self):
        """ Test case setup """
        self.global_admin = AdminFactory()
        self.instructor = User.objects.create_user('testinstructor', 'testinstructor+courses@edx.org', 'foo')
        self.staff = User.objects.create_user('teststaff', 'teststaff+courses@edx.org', 'foo')
        self.course_key = SlashSeparatedCourseKey('mitX', '101', 'test')

    def test_get_user_role_instructor(self):
        """
        Verifies if user is instructor.
        """
        add_users(self.global_admin, CourseInstructorRole(self.course_key), self.instructor)
        self.assertEqual(
            'instructor',
            get_user_role(self.instructor, self.course_key)
        )
        add_users(self.global_admin, CourseStaffRole(self.course_key), self.staff)
        self.assertEqual(
            'instructor',
            get_user_role(self.instructor, self.course_key)
        )

    def test_get_user_role_staff(self):
        """
        Verifies if user is staff.
        """
        add_users(self.global_admin, CourseStaffRole(self.course_key), self.staff)
        self.assertEqual(
            'staff',
            get_user_role(self.staff, self.course_key)
        )

########NEW FILE########
__FILENAME__ = test_assets
"""
Unit tests for the asset upload endpoint.
"""

# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0212

from datetime import datetime
from io import BytesIO
from pytz import UTC
import json
from contentstore.tests.utils import CourseTestCase
from contentstore.views import assets
from contentstore.utils import reverse_course_url
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_importer import import_from_xml
from django.test.utils import override_settings
from xmodule.modulestore.locations import SlashSeparatedCourseKey, AssetLocation


class AssetsTestCase(CourseTestCase):
    """
    Parent class for all asset tests.
    """
    def setUp(self):
        super(AssetsTestCase, self).setUp()
        self.url = reverse_course_url('assets_handler', self.course.id)

    def upload_asset(self, name="asset-1"):
        f = BytesIO(name)
        f.name = name + ".txt"
        return self.client.post(self.url, {"name": name, "file": f})


class BasicAssetsTestCase(AssetsTestCase):
    def test_basic(self):
        resp = self.client.get(self.url, HTTP_ACCEPT='text/html')
        self.assertEquals(resp.status_code, 200)

    def test_static_url_generation(self):

        course_key = SlashSeparatedCourseKey('org', 'class', 'run')
        location = course_key.make_asset_key('asset', 'my_file_name.jpg')
        path = StaticContent.get_static_path_from_location(location)
        self.assertEquals(path, '/static/my_file_name.jpg')

    def test_pdf_asset(self):
        module_store = modulestore('direct')
        _, course_items = import_from_xml(
            module_store,
            'common/test/data/',
            ['toy'],
            static_content_store=contentstore(),
            verbose=True
        )
        course = course_items[0]
        url = reverse_course_url('assets_handler', course.id)

        # Test valid contentType for pdf asset (textbook.pdf)
        resp = self.client.get(url, HTTP_ACCEPT='application/json')
        self.assertContains(resp, "/c4x/edX/toy/asset/textbook.pdf")
        asset_location = AssetLocation.from_deprecated_string('/c4x/edX/toy/asset/textbook.pdf')
        content = contentstore().find(asset_location)
        # Check after import textbook.pdf has valid contentType ('application/pdf')

        # Note: Actual contentType for textbook.pdf in asset.json is 'text/pdf'
        self.assertEqual(content.content_type, 'application/pdf')


class PaginationTestCase(AssetsTestCase):
    """
    Tests the pagination of assets returned from the REST API.
    """
    def test_json_responses(self):
        self.upload_asset("asset-1")
        self.upload_asset("asset-2")
        self.upload_asset("asset-3")

        # Verify valid page requests
        self.assert_correct_asset_response(self.url, 0, 3, 3)
        self.assert_correct_asset_response(self.url + "?page_size=2", 0, 2, 3)
        self.assert_correct_asset_response(self.url + "?page_size=2&page=1", 2, 1, 3)
        self.assert_correct_sort_response(self.url, 'date_added', 'asc')
        self.assert_correct_sort_response(self.url, 'date_added', 'desc')
        self.assert_correct_sort_response(self.url, 'display_name', 'asc')
        self.assert_correct_sort_response(self.url, 'display_name', 'desc')

        # Verify querying outside the range of valid pages
        self.assert_correct_asset_response(self.url + "?page_size=2&page=-1", 0, 2, 3)
        self.assert_correct_asset_response(self.url + "?page_size=2&page=2", 2, 1, 3)
        self.assert_correct_asset_response(self.url + "?page_size=3&page=1", 0, 3, 3)

    def assert_correct_asset_response(self, url, expected_start, expected_length, expected_total):
        resp = self.client.get(url, HTTP_ACCEPT='application/json')
        json_response = json.loads(resp.content)
        assets = json_response['assets']
        self.assertEquals(json_response['start'], expected_start)
        self.assertEquals(len(assets), expected_length)
        self.assertEquals(json_response['totalCount'], expected_total)

    def assert_correct_sort_response(self, url, sort, direction):
        resp = self.client.get(url + '?sort=' + sort + '&direction=' + direction, HTTP_ACCEPT='application/json')
        json_response = json.loads(resp.content)
        assets = json_response['assets']
        name1 = assets[0][sort]
        name2 = assets[1][sort]
        name3 = assets[2][sort]
        if direction == 'asc':
            self.assertLessEqual(name1, name2)
            self.assertLessEqual(name2, name3)
        else:
            self.assertGreaterEqual(name1, name2)
            self.assertGreaterEqual(name2, name3)


class UploadTestCase(AssetsTestCase):
    """
    Unit tests for uploading a file
    """
    def setUp(self):
        super(UploadTestCase, self).setUp()
        self.url = reverse_course_url('assets_handler', self.course.id)

    def test_happy_path(self):
        resp = self.upload_asset()
        self.assertEquals(resp.status_code, 200)

    def test_no_file(self):
        resp = self.client.post(self.url, {"name": "file.txt"}, "application/json")
        self.assertEquals(resp.status_code, 400)


class AssetToJsonTestCase(AssetsTestCase):
    """
    Unit test for transforming asset information into something
    we can send out to the client via JSON.
    """
    @override_settings(LMS_BASE="lms_base_url")
    def test_basic(self):
        upload_date = datetime(2013, 6, 1, 10, 30, tzinfo=UTC)

        course_key = SlashSeparatedCourseKey('org', 'class', 'run')
        location = course_key.make_asset_key('asset', 'my_file_name.jpg')
        thumbnail_location = course_key.make_asset_key('thumbnail', 'my_file_name_thumb.jpg')

        output = assets._get_asset_json("my_file", upload_date, location, thumbnail_location, True)

        self.assertEquals(output["display_name"], "my_file")
        self.assertEquals(output["date_added"], "Jun 01, 2013 at 10:30 UTC")
        self.assertEquals(output["url"], "/c4x/org/class/asset/my_file_name.jpg")
        self.assertEquals(output["external_url"], "lms_base_url/c4x/org/class/asset/my_file_name.jpg")
        self.assertEquals(output["portable_url"], "/static/my_file_name.jpg")
        self.assertEquals(output["thumbnail"], "/c4x/org/class/thumbnail/my_file_name_thumb.jpg")
        self.assertEquals(output["id"], unicode(location))
        self.assertEquals(output['locked'], True)

        output = assets._get_asset_json("name", upload_date, location, None, False)
        self.assertIsNone(output["thumbnail"])


class LockAssetTestCase(AssetsTestCase):
    """
    Unit test for locking and unlocking an asset.
    """

    def test_locking(self):
        """
        Tests a simple locking and unlocking of an asset in the toy course.
        """
        def verify_asset_locked_state(locked):
            """ Helper method to verify lock state in the contentstore """
            asset_location = StaticContent.get_location_from_path('/c4x/edX/toy/asset/sample_static.txt')
            content = contentstore().find(asset_location)
            self.assertEqual(content.locked, locked)

        def post_asset_update(lock, course):
            """ Helper method for posting asset update. """
            upload_date = datetime(2013, 6, 1, 10, 30, tzinfo=UTC)
            asset_location = course.id.make_asset_key('asset', 'sample_static.txt')
            url = reverse_course_url('assets_handler', course.id, kwargs={'asset_key_string': unicode(asset_location)})

            resp = self.client.post(
                url,
                json.dumps(assets._get_asset_json("sample_static.txt", upload_date, asset_location, None, lock)),
                "application/json"
            )
            self.assertEqual(resp.status_code, 201)
            return json.loads(resp.content)

        # Load the toy course.
        module_store = modulestore('direct')
        _, course_items = import_from_xml(
            module_store,
            'common/test/data/',
            ['toy'],
            static_content_store=contentstore(),
            verbose=True
        )
        course = course_items[0]
        verify_asset_locked_state(False)

        # Lock the asset
        resp_asset = post_asset_update(True, course)
        self.assertTrue(resp_asset['locked'])
        verify_asset_locked_state(True)

        # Unlock the asset
        resp_asset = post_asset_update(False, course)
        self.assertFalse(resp_asset['locked'])
        verify_asset_locked_state(False)

########NEW FILE########
__FILENAME__ = test_checklists
""" Unit tests for checklist methods in views.py. """
from contentstore.utils import get_modulestore, reverse_course_url
from contentstore.views.checklist import expand_checklist_action_url
from xmodule.modulestore.tests.factories import CourseFactory

import json
from contentstore.tests.utils import CourseTestCase


class ChecklistTestCase(CourseTestCase):
    """ Test for checklist get and put methods. """
    def setUp(self):
        """ Creates the test course. """
        super(ChecklistTestCase, self).setUp()
        self.course = CourseFactory.create(org='mitX', number='333', display_name='Checklists Course')
        self.checklists_url = self.get_url()

    def get_url(self, checklist_index=None):
        url_args = {'checklist_index': checklist_index} if checklist_index else None
        return reverse_course_url('checklists_handler', self.course.id, kwargs=url_args)

    def get_persisted_checklists(self):
        """ Returns the checklists as persisted in the modulestore. """
        modulestore = get_modulestore(self.course.location)
        return modulestore.get_item(self.course.location).checklists

    def compare_checklists(self, persisted, request):
        """
        Handles url expansion as possible difference and descends into guts
        """
        self.assertEqual(persisted['short_description'], request['short_description'])
        expanded_checklist = expand_checklist_action_url(self.course, persisted)
        for pers, req in zip(expanded_checklist['items'], request['items']):
            self.assertEqual(pers['short_description'], req['short_description'])
            self.assertEqual(pers['long_description'], req['long_description'])
            self.assertEqual(pers['is_checked'], req['is_checked'])
            self.assertEqual(pers['action_url'], req['action_url'])
            self.assertEqual(pers['action_text'], req['action_text'])
            self.assertEqual(pers['action_external'], req['action_external'])

    def test_get_checklists(self):
        """ Tests the get checklists method and URL expansion. """
        response = self.client.get(self.checklists_url)
        self.assertContains(response, "Getting Started With Studio")
        # Verify expansion of action URL happened.
        self.assertContains(response, 'course_team/slashes:mitX+333+Checklists_Course')
        # Verify persisted checklist does NOT have expanded URL.
        checklist_0 = self.get_persisted_checklists()[0]
        self.assertEqual('ManageUsers', get_action_url(checklist_0, 0))
        payload = response.content

        # Now delete the checklists from the course and verify they get repopulated (for courses
        # created before checklists were introduced).
        self.course.checklists = None
        # Save the changed `checklists` to the underlying KeyValueStore before updating the modulestore
        self.course.save()
        modulestore = get_modulestore(self.course.location)
        modulestore.update_item(self.course, self.user.id)
        self.assertEqual(self.get_persisted_checklists(), None)
        response = self.client.get(self.checklists_url)
        self.assertEqual(payload, response.content)

    def test_get_checklists_html(self):
        """ Tests getting the HTML template for the checklists page). """
        response = self.client.get(self.checklists_url, HTTP_ACCEPT='text/html')
        self.assertContains(response, "Getting Started With Studio")
        # The HTML generated will define the handler URL (for use by the Backbone model).
        self.assertContains(response, self.checklists_url)

    def test_update_checklists_no_index(self):
        """ No checklist index, should return all of them. """
        returned_checklists = json.loads(self.client.get(self.checklists_url).content)
        # Verify that persisted checklists do not have expanded action URLs.
        # compare_checklists will verify that returned_checklists DO have expanded action URLs.
        pers = self.get_persisted_checklists()
        self.assertEqual('CourseOutline', get_first_item(pers[1]).get('action_url'))
        for pay, resp in zip(pers, returned_checklists):
            self.compare_checklists(pay, resp)

    def test_update_checklists_index_ignored_on_get(self):
        """ Checklist index ignored on get. """
        update_url = self.get_url(1)

        returned_checklists = json.loads(self.client.get(update_url).content)
        for pay, resp in zip(self.get_persisted_checklists(), returned_checklists):
            self.compare_checklists(pay, resp)

    def test_update_checklists_post_no_index(self):
        """ No checklist index, will error on post. """
        response = self.client.post(self.checklists_url)
        self.assertContains(response, 'Could not save checklist', status_code=400)

    def test_update_checklists_index_out_of_range(self):
        """ Checklist index out of range, will error on post. """
        update_url = self.get_url(100)

        response = self.client.post(update_url)
        self.assertContains(response, 'Could not save checklist', status_code=400)

    def test_update_checklists_index(self):
        """ Check that an update of a particular checklist works. """
        update_url = self.get_url(1)

        payload = self.course.checklists[1]
        self.assertFalse(get_first_item(payload).get('is_checked'))
        self.assertEqual('CourseOutline', get_first_item(payload).get('action_url'))
        get_first_item(payload)['is_checked'] = True

        returned_checklist = json.loads(self.client.ajax_post(update_url, payload).content)
        self.assertTrue(get_first_item(returned_checklist).get('is_checked'))
        persisted_checklist = self.get_persisted_checklists()[1]
        # Verify that persisted checklist does not have expanded action URLs.
        # compare_checklists will verify that returned_checklist DOES have expanded action URLs.
        self.assertEqual('CourseOutline', get_first_item(persisted_checklist).get('action_url'))
        self.compare_checklists(persisted_checklist, returned_checklist)

    def test_update_checklists_delete_unsupported(self):
        """ Delete operation is not supported. """
        update_url = self.get_url(100)
        response = self.client.delete(update_url)
        self.assertEqual(response.status_code, 405)

    def test_expand_checklist_action_url(self):
        """
        Tests the method to expand checklist action url.
        """

        def test_expansion(checklist, index, stored, expanded):
            """
            Tests that the expected expanded value is returned for the item at the given index.

            Also verifies that the original checklist is not modified.
            """
            self.assertEqual(get_action_url(checklist, index), stored)
            expanded_checklist = expand_checklist_action_url(self.course, checklist)
            self.assertEqual(get_action_url(expanded_checklist, index), expanded)
            # Verify no side effect in the original list.
            self.assertEqual(get_action_url(checklist, index), stored)

        test_expansion(self.course.checklists[0], 0, 'ManageUsers', '/course_team/slashes:mitX+333+Checklists_Course/')
        test_expansion(self.course.checklists[1], 1, 'CourseOutline', '/course/slashes:mitX+333+Checklists_Course')
        test_expansion(self.course.checklists[2], 0, 'http://help.edge.edx.org/', 'http://help.edge.edx.org/')


def get_first_item(checklist):
    """ Returns the first item from the checklist. """
    return checklist['items'][0]


def get_action_url(checklist, index):
    """
    Returns the action_url for the item at the specified index in the given checklist.
    """
    return checklist['items'][index]['action_url']

########NEW FILE########
__FILENAME__ = test_container_page
"""
Unit tests for the container page.
"""

import re
from contentstore.utils import compute_publish_state, PublishState
from contentstore.views.tests.utils import StudioPageTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import ItemFactory


class ContainerPageTestCase(StudioPageTestCase):
    """
    Unit tests for the container page.
    """

    container_view = 'container_preview'
    reorderable_child_view = 'reorderable_container_child_preview'

    def setUp(self):
        super(ContainerPageTestCase, self).setUp()
        self.vertical = ItemFactory.create(parent_location=self.sequential.location,
                                           category='vertical', display_name='Unit')
        self.html = ItemFactory.create(parent_location=self.vertical.location,
                                        category="html", display_name="HTML")
        self.child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                  category='split_test', display_name='Split Test')
        self.child_vertical = ItemFactory.create(parent_location=self.child_container.location,
                                                 category='vertical', display_name='Child Vertical')
        self.video = ItemFactory.create(parent_location=self.child_vertical.location,
                                        category="video", display_name="My Video")

    def test_container_html(self):
        self._test_html_content(
            self.child_container,
            expected_section_tag=(
                '<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
                'data-locator="{0}" data-course-key="{0.course_key}">'.format(self.child_container.location)
            ),
            expected_breadcrumbs=(
                r'<a href="/unit/{}"\s*'
                r'class="navigation-link navigation-parent">Unit</a>\s*'
                r'<a href="#" class="navigation-link navigation-current">Split Test</a>'
            ).format(re.escape(unicode(self.vertical.location)))
        )

    def test_container_on_container_html(self):
        """
        Create the scenario of an xblock with children (non-vertical) on the container page.
        This should create a container page that is a child of another container page.
        """
        published_container = ItemFactory.create(
            parent_location=self.child_container.location,
            category="wrapper", display_name="Wrapper"
        )
        ItemFactory.create(
            parent_location=published_container.location,
            category="html", display_name="Child HTML"
        )

        def test_container_html(xblock):
            self._test_html_content(
                xblock,
                expected_section_tag=(
                    '<section class="wrapper-xblock level-page is-hidden studio-xblock-wrapper" '
                    'data-locator="{0}" data-course-key="{0.course_key}">'.format(published_container.location)
                ),
                expected_breadcrumbs=(
                    r'<a href="/unit/{unit}"\s*'
                    r'class="navigation-link navigation-parent">Unit</a>\s*'
                    r'<a href="/container/{split_test}"\s*'
                    r'class="navigation-link navigation-parent">Split Test</a>\s*'
                    r'<a href="#" class="navigation-link navigation-current">Wrapper</a>'
                ).format(
                    unit=re.escape(unicode(self.vertical.location)),
                    split_test=re.escape(unicode(self.child_container.location))
                )
            )

        # Test the published version of the container
        test_container_html(published_container)

        # Now make the unit and its children into a draft and validate the container again
        modulestore('draft').convert_to_draft(self.vertical.location)
        modulestore('draft').convert_to_draft(self.child_vertical.location)
        draft_container = modulestore('draft').convert_to_draft(published_container.location)
        test_container_html(draft_container)

    def _test_html_content(self, xblock, expected_section_tag, expected_breadcrumbs):
        """
        Get the HTML for a container page and verify the section tag is correct
        and the breadcrumbs trail is correct.
        """
        html = self.get_page_html(xblock)
        publish_state = compute_publish_state(xblock)
        self.assertIn(expected_section_tag, html)
        # Verify the navigation link at the top of the page is correct.
        self.assertRegexpMatches(html, expected_breadcrumbs)

        # Verify the link that allows users to change publish status.
        expected_message = None
        if publish_state == PublishState.public:
            expected_message = 'you need to edit unit <a href="/unit/{}">Unit</a> as a draft.'
        else:
            expected_message = 'your changes will be published with unit <a href="/unit/{}">Unit</a>.'
        expected_unit_link = expected_message.format(self.vertical.location)
        self.assertIn(expected_unit_link, html)

    def test_public_container_preview_html(self):
        """
        Verify that a public xblock's container preview returns the expected HTML.
        """
        self.validate_preview_html(self.vertical, self.container_view,
                                   can_edit=False, can_reorder=False, can_add=False)
        self.validate_preview_html(self.child_container, self.container_view,
                                   can_edit=False, can_reorder=False, can_add=False)
        self.validate_preview_html(self.child_vertical, self.reorderable_child_view,
                                   can_edit=False, can_reorder=False, can_add=False)

    def test_draft_container_preview_html(self):
        """
        Verify that a draft xblock's container preview returns the expected HTML.
        """
        draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
        draft_child_container = modulestore('draft').convert_to_draft(self.child_container.location)
        draft_child_vertical = modulestore('draft').convert_to_draft(self.child_vertical.location)
        self.validate_preview_html(draft_unit, self.container_view,
                                   can_edit=True, can_reorder=True, can_add=True)
        self.validate_preview_html(draft_child_container, self.container_view,
                                   can_edit=True, can_reorder=True, can_add=True)
        self.validate_preview_html(draft_child_vertical, self.reorderable_child_view,
                                   can_edit=True, can_reorder=True, can_add=True)

    def test_public_child_container_preview_html(self):
        """
        Verify that a public container rendered as a child of the container page returns the expected HTML.
        """
        empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                   category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=empty_child_container.location,
                           category='html', display_name='Split Child')
        self.validate_preview_html(empty_child_container, self.reorderable_child_view,
                                   can_reorder=False, can_edit=False, can_add=False)

    def test_draft_child_container_preview_html(self):
        """
        Verify that a draft container rendered as a child of the container page returns the expected HTML.
        """
        empty_child_container = ItemFactory.create(parent_location=self.vertical.location,
                                                   category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=empty_child_container.location,
                           category='html', display_name='Split Child')
        modulestore('draft').convert_to_draft(self.vertical.location)
        draft_empty_child_container = modulestore('draft').convert_to_draft(empty_child_container.location)
        self.validate_preview_html(draft_empty_child_container, self.reorderable_child_view,
                                   can_reorder=True, can_edit=False, can_add=False)

########NEW FILE########
__FILENAME__ = test_course_index
"""
Unit tests for getting the list of courses and the course outline.
"""
import json
import lxml

from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.locator import Locator


class TestCourseIndex(CourseTestCase):
    """
    Unit tests for getting the list of courses and the course outline.
    """
    def setUp(self):
        """
        Add a course with odd characters in the fields
        """
        super(TestCourseIndex, self).setUp()
        # had a problem where index showed course but has_access failed to retrieve it for non-staff
        self.odd_course = CourseFactory.create(
            org='test.org_1-2',
            number='test-2.3_course',
            display_name='dotted.course.name-2',
        )

    def check_index_and_outline(self, authed_client):
        """
        Test getting the list of courses and then pulling up their outlines
        """
        index_url = '/course/'
        index_response = authed_client.get(index_url, {}, HTTP_ACCEPT='text/html')
        parsed_html = lxml.html.fromstring(index_response.content)
        course_link_eles = parsed_html.find_class('course-link')
        self.assertGreaterEqual(len(course_link_eles), 2)
        for link in course_link_eles:
            self.assertRegexpMatches(
                link.get("href"),
                'course/slashes:{0}'.format(Locator.ALLOWED_ID_CHARS)
            )
            # now test that url
            outline_response = authed_client.get(link.get("href"), {}, HTTP_ACCEPT='text/html')
            # ensure it has the expected 2 self referential links
            outline_parsed = lxml.html.fromstring(outline_response.content)
            outline_link = outline_parsed.find_class('course-link')[0]
            self.assertEqual(outline_link.get("href"), link.get("href"))
            course_menu_link = outline_parsed.find_class('nav-course-courseware-outline')[0]
            self.assertEqual(course_menu_link.find("a").get("href"), link.get("href"))

    def test_is_staff_access(self):
        """
        Test that people with is_staff see the courses and can navigate into them
        """
        self.check_index_and_outline(self.client)

    def test_negative_conditions(self):
        """
        Test the error conditions for the access
        """
        outline_url = reverse_course_url('course_handler', self.course.id)
        # register a non-staff member and try to delete the course branch
        non_staff_client, _ = self.create_non_staff_authed_user_client()
        response = non_staff_client.delete(outline_url, {}, HTTP_ACCEPT='application/json')
        self.assertEqual(response.status_code, 403)

    def test_course_staff_access(self):
        """
        Make and register course_staff and ensure they can access the courses
        """
        course_staff_client, course_staff = self.create_non_staff_authed_user_client()
        for course in [self.course, self.odd_course]:
            permission_url = reverse_course_url('course_team_handler', course.id, kwargs={'email': course_staff.email})

            self.client.post(
                permission_url,
                data=json.dumps({"role": "staff"}),
                content_type="application/json",
                HTTP_ACCEPT="application/json",
            )

        # test access
        self.check_index_and_outline(course_staff_client)

    def test_json_responses(self):
        outline_url = reverse_course_url('course_handler', self.course.id)
        chapter = ItemFactory.create(parent_location=self.course.location, category='chapter', display_name="Week 1")
        lesson = ItemFactory.create(parent_location=chapter.location, category='sequential', display_name="Lesson 1")
        subsection = ItemFactory.create(parent_location=lesson.location, category='vertical', display_name='Subsection 1')
        ItemFactory.create(parent_location=subsection.location, category="video", display_name="My Video")

        resp = self.client.get(outline_url, HTTP_ACCEPT='application/json')
        json_response = json.loads(resp.content)

        # First spot check some values in the root response
        self.assertEqual(json_response['category'], 'course')
        self.assertEqual(json_response['id'], 'location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')
        self.assertEqual(json_response['display_name'], 'Robot Super Course')
        self.assertTrue(json_response['is_container'])
        self.assertFalse(json_response['is_draft'])

        # Now verify the first child
        children = json_response['children']
        self.assertTrue(len(children) > 0)
        first_child_response = children[0]
        self.assertEqual(first_child_response['category'], 'chapter')
        self.assertEqual(first_child_response['id'], 'location:MITx+999+Robot_Super_Course+chapter+Week_1')
        self.assertEqual(first_child_response['display_name'], 'Week 1')
        self.assertTrue(first_child_response['is_container'])
        self.assertFalse(first_child_response['is_draft'])
        self.assertTrue(len(first_child_response['children']) > 0)

        # Finally, validate the entire response for consistency
        self.assert_correct_json_response(json_response)

    def assert_correct_json_response(self, json_response):
        """
        Asserts that the JSON response is syntactically consistent
        """
        self.assertIsNotNone(json_response['display_name'])
        self.assertIsNotNone(json_response['id'])
        self.assertIsNotNone(json_response['category'])
        self.assertIsNotNone(json_response['is_draft'])
        self.assertIsNotNone(json_response['is_container'])
        if json_response['is_container']:
            for child_response in json_response['children']:
                self.assert_correct_json_response(child_response)
        else:
            self.assertFalse('children' in json_response)

########NEW FILE########
__FILENAME__ = test_course_updates
"""
unit tests for course_info views and models.
"""
import json

from contentstore.tests.test_course_settings import CourseTestCase
from contentstore.utils import reverse_course_url, reverse_usage_url
from xmodule.modulestore.locations import Location, SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore


class CourseUpdateTest(CourseTestCase):

    def create_update_url(self, provided_id=None, course_key=None):
        if course_key is None:
            course_key = self.course.id
        kwargs = {'provided_id': str(provided_id)} if provided_id else None
        return reverse_course_url('course_info_update_handler', course_key, kwargs=kwargs)

    '''The do all and end all of unit test cases.'''
    def test_course_update(self):
        '''Go through each interface and ensure it works.'''
        def get_response(content, date):
            """
            Helper method for making call to server and returning response.

            Does not supply a provided_id.
            """
            payload = {'content': content, 'date': date}
            url = self.create_update_url()

            resp = self.client.ajax_post(url, payload)
            self.assertContains(resp, '', status_code=200)

            return json.loads(resp.content)

        resp = self.client.get_html(
            reverse_course_url('course_info_handler', self.course.id)
        )
        self.assertContains(resp, 'Course Updates', status_code=200)

        init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
        content = init_content + '</iframe>'
        payload = get_response(content, 'January 8, 2013')
        self.assertHTMLEqual(payload['content'], content)

        first_update_url = self.create_update_url(provided_id=payload['id'])
        content += '<div>div <p>p<br/></p></div>'
        payload['content'] = content
        # POST requests were coming in w/ these header values causing an error; so, repro error here
        resp = self.client.ajax_post(
            first_update_url, payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
        )

        self.assertHTMLEqual(content, json.loads(resp.content)['content'],
                             "iframe w/ div")
        # refetch using provided id
        refetched = self.client.get_json(first_update_url)
        self.assertHTMLEqual(
            content, json.loads(refetched.content)['content'], "get w/ provided id"
        )

        # now put in an evil update
        content = '<ol/>'
        payload = get_response(content, 'January 11, 2013')
        self.assertHTMLEqual(content, payload['content'], "self closing ol")

        course_update_url = self.create_update_url()
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        self.assertTrue(len(payload) == 2)

        # try json w/o required fields
        self.assertContains(
            self.client.ajax_post(course_update_url, {'garbage': 1}),
            'Failed to save', status_code=400
        )

        # test an update with text in the tail of the header
        content = 'outside <strong>inside</strong> after'
        payload = get_response(content, 'June 22, 2000')
        self.assertHTMLEqual(content, payload['content'], "text outside tag")

        # now try to update a non-existent update
        content = 'blah blah'
        payload = {'content': content, 'date': 'January 21, 2013'}
        self.assertContains(
            self.client.ajax_post(course_update_url + '9', payload),
            'Failed to save', status_code=400
        )

        # update w/ malformed html
        content = '<garbage tag No closing brace to force <span>error</span>'
        payload = {'content': content,
                   'date': 'January 11, 2013'}

        self.assertContains(
            self.client.ajax_post(course_update_url, payload),
            '<garbage'
        )

        # set to valid html which would break an xml parser
        content = "<p><br><br></p>"
        payload = get_response(content, 'January 11, 2013')
        self.assertHTMLEqual(content, payload['content'])

        # now try to delete a non-existent update
        self.assertContains(self.client.delete(course_update_url + '19'), "delete", status_code=400)

        # now delete a real update
        content = 'blah blah'
        payload = get_response(content, 'January 28, 2013')
        this_id = payload['id']
        self.assertHTMLEqual(content, payload['content'], "single iframe")
        # first count the entries
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        before_delete = len(payload)

        url = self.create_update_url(provided_id=this_id)
        resp = self.client.delete(url)
        payload = json.loads(resp.content)
        self.assertTrue(len(payload) == before_delete - 1)

    def test_course_updates_compatibility(self):
        '''
        Test that course updates doesn't break on old data (content in 'data' field).
        Note: new data will save as list in 'items' field.
        '''
        # get the updates and populate 'data' field with some data.
        location = self.course.id.make_usage_key('course_info', 'updates')
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)
        update_date = u"January 23, 2014"
        update_content = u"Hello world!"
        update_data = u"<ol><li><h2>" + update_date + "</h2>" + update_content + "</li></ol>"
        course_updates.data = update_data
        modulestore('direct').update_item(course_updates, self.user)

        # test getting all updates list
        course_update_url = self.create_update_url()
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        self.assertEqual(payload, [{u'date': update_date, u'content': update_content, u'id': 1}])
        self.assertTrue(len(payload) == 1)

        # test getting single update item

        first_update_url = self.create_update_url(provided_id=payload[0]['id'])
        resp = self.client.get_json(first_update_url)
        payload = json.loads(resp.content)
        self.assertEqual(payload, {u'date': u'January 23, 2014', u'content': u'Hello world!', u'id': 1})
        self.assertHTMLEqual(update_date, payload['date'])
        self.assertHTMLEqual(update_content, payload['content'])

        # test that while updating it converts old data (with string format in 'data' field)
        # to new data (with list format in 'items' field) and respectively updates 'data' field.
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items, [])
        # now try to update first update item
        update_content = 'Testing'
        payload = {'content': update_content, 'date': update_date}
        resp = self.client.ajax_post(
            course_update_url + '1', payload, HTTP_X_HTTP_METHOD_OVERRIDE="PUT", REQUEST_METHOD="POST"
        )
        self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
        # course_updates 'data' field should update accordingly
        update_data = u"<section><article><h2>{date}</h2>{content}</article></section>".format(date=update_date, content=update_content)
        self.assertEqual(course_updates.data, update_data)

        # test delete course update item (soft delete)
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items, [{u'date': update_date, u'content': update_content, u'id': 1}])
        # now try to delete first update item
        resp = self.client.delete(course_update_url + '1')
        self.assertEqual(json.loads(resp.content), [])
        # confirm that course update is soft deleted ('status' flag set to 'deleted') in db
        course_updates = modulestore('direct').get_item(location)
        self.assertEqual(course_updates.items,
                         [{u'date': update_date, u'content': update_content, u'id': 1, u'status': 'deleted'}])

        # now try to get deleted update
        resp = self.client.get_json(course_update_url + '1')
        payload = json.loads(resp.content)
        self.assertEqual(payload.get('error'), u"Course update not found.")
        self.assertEqual(resp.status_code, 404)

        # now check that course update don't munges html
        update_content = u"""&lt;problem>
                           &lt;p>&lt;/p>
                           &lt;multiplechoiceresponse>
                           <pre>&lt;problem>
                               &lt;p>&lt;/p></pre>
                           <div><foo>bar</foo></div>"""
        payload = {'content': update_content, 'date': update_date}
        resp = self.client.ajax_post(
            course_update_url, payload, REQUEST_METHOD="POST"
        )
        self.assertHTMLEqual(update_content, json.loads(resp.content)['content'])

    def test_no_ol_course_update(self):
        '''Test trying to add to a saved course_update which is not an ol.'''
        # get the updates and set to something wrong
        location = self.course.id.make_usage_key('course_info', 'updates')
        modulestore('direct').create_and_save_xmodule(location)
        course_updates = modulestore('direct').get_item(location)
        course_updates.data = 'bad news'
        modulestore('direct').update_item(course_updates, self.user.id)

        init_content = '<iframe width="560" height="315" src="http://www.youtube.com/embed/RocY-Jd93XU" frameborder="0">'
        content = init_content + '</iframe>'
        payload = {'content': content, 'date': 'January 8, 2013'}

        course_update_url = self.create_update_url()
        resp = self.client.ajax_post(course_update_url, payload)

        payload = json.loads(resp.content)

        self.assertHTMLEqual(payload['content'], content)

        # now confirm that the bad news and the iframe make up single update
        resp = self.client.get_json(course_update_url)
        payload = json.loads(resp.content)
        self.assertTrue(len(payload) == 1)

    def test_post_course_update(self):
        """
        Test that a user can successfully post on course updates and handouts of a course
        """
        course_key = SlashSeparatedCourseKey('Org1', 'Course_1', 'Run_1')
        course_update_url = self.create_update_url(course_key=course_key)

        # create a course via the view handler
        self.client.ajax_post(course_update_url)

        block = u'updates'
        content = u"Sample update"
        payload = {'content': content, 'date': 'January 8, 2013'}
        resp = self.client.ajax_post(course_update_url, payload)

        # check that response status is 200 not 400
        self.assertEqual(resp.status_code, 200)

        payload = json.loads(resp.content)
        self.assertHTMLEqual(payload['content'], content)

        updates_location = self.course.id.make_usage_key('course_info', 'updates')
        self.assertTrue(isinstance(updates_location, Location))
        self.assertEqual(updates_location.name, block)

        # check posting on handouts
        handouts_location = self.course.id.make_usage_key('course_info', 'handouts')
        course_handouts_url = reverse_usage_url('xblock_handler', handouts_location)

        content = u"Sample handout"
        payload = {'data': content}
        resp = self.client.ajax_post(course_handouts_url, payload)
        # check that response status is 200 not 500
        self.assertEqual(resp.status_code, 200)

        payload = json.loads(resp.content)
        self.assertHTMLEqual(payload['data'], content)

########NEW FILE########
__FILENAME__ = test_helpers
"""
Unit tests for helpers.py.
"""

from contentstore.tests.utils import CourseTestCase
from contentstore.views.helpers import xblock_studio_url
from xmodule.modulestore.tests.factories import ItemFactory


class HelpersTestCase(CourseTestCase):
    """
    Unit tests for helpers.py.
    """
    def test_xblock_studio_url(self):

        # Verify course URL
        self.assertEqual(xblock_studio_url(self.course),
                         u'/course/slashes:MITx+999+Robot_Super_Course')

        # Verify chapter URL
        chapter = ItemFactory.create(parent_location=self.course.location, category='chapter',
                                     display_name="Week 1")
        self.assertIsNone(xblock_studio_url(chapter))

        # Verify lesson URL
        sequential = ItemFactory.create(parent_location=chapter.location, category='sequential',
                                        display_name="Lesson 1")
        self.assertIsNone(xblock_studio_url(sequential))

        # Verify vertical URL
        vertical = ItemFactory.create(parent_location=sequential.location, category='vertical',
                                      display_name='Unit')
        self.assertEqual(xblock_studio_url(vertical),
                         u'/unit/location:MITx+999+Robot_Super_Course+vertical+Unit')

        # Verify child vertical URL
        child_vertical = ItemFactory.create(parent_location=vertical.location, category='vertical',
                                            display_name='Child Vertical')
        self.assertEqual(xblock_studio_url(child_vertical),
                         u'/container/location:MITx+999+Robot_Super_Course+vertical+Child_Vertical')

        # Verify video URL
        video = ItemFactory.create(parent_location=child_vertical.location, category="video",
                                   display_name="My Video")
        self.assertIsNone(xblock_studio_url(video))

########NEW FILE########
__FILENAME__ = test_import_export
"""
Unit tests for course import and export
"""
import copy
import json
import logging
import os
import shutil
import tarfile
import tempfile
from path import path
from pymongo import MongoClient
from uuid import uuid4

from django.test.utils import override_settings
from django.conf import settings
from contentstore.utils import reverse_course_url

from xmodule.contentstore.django import _CONTENTSTORE
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.factories import ItemFactory

from contentstore.tests.utils import CourseTestCase
from student import auth
from student.roles import CourseInstructorRole, CourseStaffRole

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex

log = logging.getLogger(__name__)


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ImportTestCase(CourseTestCase):
    """
    Unit tests for importing a course
    """
    def setUp(self):
        super(ImportTestCase, self).setUp()
        self.url = reverse_course_url('import_handler', self.course.id)
        self.content_dir = path(tempfile.mkdtemp())

        def touch(name):
            """ Equivalent to shell's 'touch'"""
            with file(name, 'a'):
                os.utime(name, None)

        # Create tar test files -----------------------------------------------
        # OK course:
        good_dir = tempfile.mkdtemp(dir=self.content_dir)
        os.makedirs(os.path.join(good_dir, "course"))
        with open(os.path.join(good_dir, "course.xml"), "w+") as f:
            f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')

        with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f:
            f.write('<course></course>')

        self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
        with tarfile.open(self.good_tar, "w:gz") as gtar:
            gtar.add(good_dir)

        # Bad course (no 'course.xml' file):
        bad_dir = tempfile.mkdtemp(dir=self.content_dir)
        touch(os.path.join(bad_dir, "bad.xml"))
        self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
        with tarfile.open(self.bad_tar, "w:gz") as btar:
            btar.add(bad_dir)

        self.unsafe_common_dir = path(tempfile.mkdtemp(dir=self.content_dir))

    def tearDown(self):
        shutil.rmtree(self.content_dir)
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_no_coursexml(self):
        """
        Check that the response for a tar.gz import without a course.xml is
        correct.
        """
        with open(self.bad_tar) as btar:
            resp = self.client.post(
                self.url,
                {
                    "name": self.bad_tar,
                    "course-data": [btar]
                })
        self.assertEquals(resp.status_code, 415)
        # Check that `import_status` returns the appropriate stage (i.e., the
        # stage at which import failed).
        resp_status = self.client.get(
            reverse_course_url(
                'import_status_handler',
                self.course.id,
                kwargs={'filename': os.path.split(self.bad_tar)[1]}
            )
        )

        self.assertEquals(json.loads(resp_status.content)["ImportStatus"], 2)

    def test_with_coursexml(self):
        """
        Check that the response for a tar.gz import with a course.xml is
        correct.
        """
        with open(self.good_tar) as gtar:
            args = {"name": self.good_tar, "course-data": [gtar]}
            resp = self.client.post(self.url, args)

        self.assertEquals(resp.status_code, 200)

    def test_import_in_existing_course(self):
        """
        Check that course is imported successfully in existing course and users have their access roles
        """
        # Create a non_staff user and add it to course staff only
        __, nonstaff_user = self.create_non_staff_authed_user_client(authenticate=False)
        auth.add_users(self.user, CourseStaffRole(self.course.id), nonstaff_user)

        course = self.store.get_course(self.course.id)
        self.assertIsNotNone(course)
        display_name_before_import = course.display_name

        # Check that global staff user can import course
        with open(self.good_tar) as gtar:
            args = {"name": self.good_tar, "course-data": [gtar]}
            resp = self.client.post(self.url, args)
        self.assertEquals(resp.status_code, 200)

        course = self.store.get_course(self.course.id)
        self.assertIsNotNone(course)
        display_name_after_import = course.display_name

        # Check that course display name have changed after import
        self.assertNotEqual(display_name_before_import, display_name_after_import)

        # Now check that non_staff user has his same role
        self.assertFalse(CourseInstructorRole(self.course.id).has_user(nonstaff_user))
        self.assertTrue(CourseStaffRole(self.course.id).has_user(nonstaff_user))

        # Now course staff user can also successfully import course
        self.client.login(username=nonstaff_user.username, password='foo')
        with open(self.good_tar) as gtar:
            args = {"name": self.good_tar, "course-data": [gtar]}
            resp = self.client.post(self.url, args)
        self.assertEquals(resp.status_code, 200)

        # Now check that non_staff user has his same role
        self.assertFalse(CourseInstructorRole(self.course.id).has_user(nonstaff_user))
        self.assertTrue(CourseStaffRole(self.course.id).has_user(nonstaff_user))

    ## Unsafe tar methods #####################################################
    # Each of these methods creates a tarfile with a single type of unsafe
    # content.
    def _fifo_tar(self):
        """
        Tar file with FIFO
        """
        fifop = self.unsafe_common_dir / "fifo.file"
        fifo_tar = self.unsafe_common_dir / "fifo.tar.gz"
        os.mkfifo(fifop)
        with tarfile.open(fifo_tar, "w:gz") as tar:
            tar.add(fifop)

        return fifo_tar

    def _symlink_tar(self):
        """
        Tarfile with symlink to path outside directory.
        """
        outsidep = self.unsafe_common_dir / "unsafe_file.txt"
        symlinkp = self.unsafe_common_dir / "symlink.txt"
        symlink_tar = self.unsafe_common_dir / "symlink.tar.gz"
        outsidep.symlink(symlinkp)
        with tarfile.open(symlink_tar, "w:gz") as tar:
            tar.add(symlinkp)

        return symlink_tar

    def _outside_tar(self):
        """
        Tarfile with file that extracts to outside directory.

        Extracting this tarfile in directory <dir> will put its contents
        directly in <dir> (rather than <dir/tarname>).
        """
        outside_tar = self.unsafe_common_dir / "unsafe_file.tar.gz"
        with tarfile.open(outside_tar, "w:gz") as tar:
            tar.addfile(tarfile.TarInfo(str(self.content_dir / "a_file")))

        return outside_tar

    def _outside_tar2(self):
        """
        Tarfile with file that extracts to outside directory.

        The path here matches the basename (`self.unsafe_common_dir`), but
        then "cd's out". E.g. "/usr/../etc" == "/etc", but the naive basename
        of the first (but not the second) is "/usr"

        Extracting this tarfile in directory <dir> will also put its contents
        directly in <dir> (rather than <dir/tarname>).
        """
        outside_tar = self.unsafe_common_dir / "unsafe_file.tar.gz"
        with tarfile.open(outside_tar, "w:gz") as tar:
            tar.addfile(tarfile.TarInfo(str(self.unsafe_common_dir / "../a_file")))

        return outside_tar

    def test_unsafe_tar(self):
        """
        Check that safety measure work.

        This includes:
            'tarbombs' which include files or symlinks with paths
        outside or directly in the working directory,
            'special files' (character device, block device or FIFOs),

        all raise exceptions/400s.
        """

        def try_tar(tarpath):
            with open(tarpath) as tar:
                args = {"name": tarpath, "course-data": [tar]}
                resp = self.client.post(self.url, args)
            self.assertEquals(resp.status_code, 400)
            self.assertTrue("SuspiciousFileOperation" in resp.content)

        try_tar(self._fifo_tar())
        try_tar(self._symlink_tar())
        try_tar(self._outside_tar())
        try_tar(self._outside_tar2())
        # Check that `import_status` returns the appropriate stage (i.e.,
        # either 3, indicating all previous steps are completed, or 0,
        # indicating no upload in progress)
        resp_status = self.client.get(
            reverse_course_url(
                'import_status_handler',
                self.course.id,
                kwargs={'filename': os.path.split(self.good_tar)[1]}
            )
        )
        import_status = json.loads(resp_status.content)["ImportStatus"]
        self.assertIn(import_status, (0, 3))


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ExportTestCase(CourseTestCase):
    """
    Tests for export_handler.
    """
    def setUp(self):
        """
        Sets up the test course.
        """
        super(ExportTestCase, self).setUp()
        self.url = reverse_course_url('export_handler', self.course.id)

    def test_export_html(self):
        """
        Get the HTML for the page.
        """
        resp = self.client.get_html(self.url)
        self.assertEquals(resp.status_code, 200)
        self.assertContains(resp, "Export My Course Content")

    def test_export_json_unsupported(self):
        """
        JSON is unsupported.
        """
        resp = self.client.get(self.url, HTTP_ACCEPT='application/json')
        self.assertEquals(resp.status_code, 406)

    def test_export_targz(self):
        """
        Get tar.gz file, using HTTP_ACCEPT.
        """
        resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz')
        self._verify_export_succeeded(resp)

    def test_export_targz_urlparam(self):
        """
        Get tar.gz file, using URL parameter.
        """
        resp = self.client.get(self.url + '?_accept=application/x-tgz')
        self._verify_export_succeeded(resp)

    def _verify_export_succeeded(self, resp):
        """ Export success helper method. """
        self.assertEquals(resp.status_code, 200)
        self.assertTrue(resp.get('Content-Disposition').startswith('attachment'))

    def test_export_failure_top_level(self):
        """
        Export failure.
        """
        ItemFactory.create(parent_location=self.course.location, category='aawefawef')
        self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+course+Robot_Super_Course')

    def test_export_failure_subsection_level(self):
        """
        Slightly different export failure.
        """
        vertical = ItemFactory.create(parent_location=self.course.location, category='vertical', display_name='foo')
        ItemFactory.create(
            parent_location=vertical.location,
            category='aawefawef'
        )

        self._verify_export_failure(u'/unit/location:MITx+999+Robot_Super_Course+vertical+foo')

    def _verify_export_failure(self, expectedText):
        """ Export failure helper method. """
        resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz')
        self.assertEquals(resp.status_code, 200)
        self.assertIsNone(resp.get('Content-Disposition'))
        self.assertContains(resp, 'Unable to create xml for module')
        self.assertContains(resp, expectedText)

########NEW FILE########
__FILENAME__ = test_item
"""Tests for items views."""

import json
from datetime import datetime
import ddt

from mock import patch
from pytz import UTC
from webob import Response

from django.http import Http404
from django.test import TestCase
from django.test.client import RequestFactory
from django.core.urlresolvers import reverse
from contentstore.utils import reverse_usage_url

from contentstore.views.component import component_handler

from contentstore.tests.utils import CourseTestCase
from contentstore.utils import compute_publish_state, PublishState
from student.tests.factories import UserFactory
from xmodule.capa_module import CapaDescriptor
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.keys import UsageKey
from xmodule.modulestore.locations import Location


class ItemTest(CourseTestCase):
    """ Base test class for create, save, and delete """
    def setUp(self):
        super(ItemTest, self).setUp()

        self.course_key = self.course.id
        self.usage_key = self.course.location

    @staticmethod
    def get_item_from_modulestore(usage_key, draft=False):
        """
        Get the item referenced by the UsageKey from the modulestore
        """
        store = modulestore('draft') if draft else modulestore('direct')
        return store.get_item(usage_key)

    def response_usage_key(self, response):
        """
        Get the UsageKey from the response payload and verify that the status_code was 200.
        :param response:
        """
        parsed = json.loads(response.content)
        self.assertEqual(response.status_code, 200)
        return UsageKey.from_string(parsed['locator'])

    def create_xblock(self, parent_usage_key=None, display_name=None, category=None, boilerplate=None):
        data = {
            'parent_locator': unicode(self.usage_key) if parent_usage_key is None else unicode(parent_usage_key),
            'category': category
        }
        if display_name is not None:
            data['display_name'] = display_name
        if boilerplate is not None:
            data['boilerplate'] = boilerplate
        return self.client.ajax_post(reverse('contentstore.views.xblock_handler'), json.dumps(data))


class GetItem(ItemTest):
    """Tests for '/xblock' GET url."""

    def _create_vertical(self, parent_usage_key=None):
        """
        Creates a vertical, returning its UsageKey.
        """
        resp = self.create_xblock(category='vertical', parent_usage_key=parent_usage_key)
        self.assertEqual(resp.status_code, 200)
        return self.response_usage_key(resp)

    def _get_container_preview(self, usage_key):
        """
        Returns the HTML and resources required for the xblock at the specified UsageKey
        """
        preview_url = reverse_usage_url("xblock_view_handler", usage_key, {'view_name': 'container_preview'})
        resp = self.client.get(preview_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)
        resp_content = json.loads(resp.content)
        html = resp_content['html']
        self.assertTrue(html)
        resources = resp_content['resources']
        self.assertIsNotNone(resources)
        return html, resources

    def test_get_vertical(self):
        # Add a vertical
        resp = self.create_xblock(category='vertical')
        usage_key = self.response_usage_key(resp)

        # Retrieve it
        resp = self.client.get(reverse_usage_url('xblock_handler', usage_key))
        self.assertEqual(resp.status_code, 200)

    def test_get_empty_container_fragment(self):
        root_usage_key = self._create_vertical()
        html, __ = self._get_container_preview(root_usage_key)

        # Verify that the Studio wrapper is not added
        self.assertNotIn('wrapper-xblock', html)

        # Verify that the header and article tags are still added
        self.assertIn('<header class="xblock-header">', html)
        self.assertIn('<article class="xblock-render">', html)

    def test_get_container_fragment(self):
        root_usage_key = self._create_vertical()

        # Add a problem beneath a child vertical
        child_vertical_usage_key = self._create_vertical(parent_usage_key=root_usage_key)
        resp = self.create_xblock(parent_usage_key=child_vertical_usage_key, category='problem', boilerplate='multiplechoice.yaml')
        self.assertEqual(resp.status_code, 200)

        # Get the preview HTML
        html, __ = self._get_container_preview(root_usage_key)

        # Verify that the Studio nesting wrapper has been added
        self.assertIn('level-nesting', html)
        self.assertIn('<header class="xblock-header">', html)
        self.assertIn('<article class="xblock-render">', html)

        # Verify that the Studio element wrapper has been added
        self.assertIn('level-element', html)

    def test_get_container_nested_container_fragment(self):
        """
        Test the case of the container page containing a link to another container page.
        """
        # Add a wrapper with child beneath a child vertical
        root_usage_key = self._create_vertical()

        resp = self.create_xblock(parent_usage_key=root_usage_key, category="wrapper")
        self.assertEqual(resp.status_code, 200)
        wrapper_usage_key = self.response_usage_key(resp)

        resp = self.create_xblock(parent_usage_key=wrapper_usage_key, category='problem', boilerplate='multiplechoice.yaml')
        self.assertEqual(resp.status_code, 200)

        # Get the preview HTML and verify the View -> link is present.
        html, __ = self._get_container_preview(root_usage_key)
        self.assertIn('wrapper-xblock', html)
        self.assertRegexpMatches(
            html,
            # The instance of the wrapper class will have an auto-generated ID. Allow any
            # characters after wrapper.
            (r'"/container/location:MITx\+999\+Robot_Super_Course\+wrapper\+\w+" class="action-button">\s*'
             '<span class="action-button-text">View</span>')
        )

    def test_split_test(self):
        """
        Test that a split_test module renders all of its children in Studio.
        """
        root_usage_key = self._create_vertical()
        resp = self.create_xblock(category='split_test', parent_usage_key=root_usage_key)
        split_test_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=split_test_usage_key, category='html', boilerplate='announcement.yaml')
        self.assertEqual(resp.status_code, 200)
        resp = self.create_xblock(parent_usage_key=split_test_usage_key, category='html', boilerplate='zooming_image.yaml')
        self.assertEqual(resp.status_code, 200)
        html, __ = self._get_container_preview(split_test_usage_key)
        self.assertIn('Announcement', html)
        self.assertIn('Zooming', html)


class DeleteItem(ItemTest):
    """Tests for '/xblock' DELETE url."""
    def test_delete_static_page(self):
        # Add static tab
        resp = self.create_xblock(category='static_tab')
        usage_key = self.response_usage_key(resp)

        # Now delete it. There was a bug that the delete was failing (static tabs do not exist in draft modulestore).
        resp = self.client.delete(reverse_usage_url('xblock_handler', usage_key))
        self.assertEqual(resp.status_code, 204)


class TestCreateItem(ItemTest):
    """
    Test the create_item handler thoroughly
    """
    def test_create_nicely(self):
        """
        Try the straightforward use cases
        """
        # create a chapter
        display_name = 'Nicely created'
        resp = self.create_xblock(display_name=display_name, category='chapter')

        # get the new item and check its category and display_name
        chap_usage_key = self.response_usage_key(resp)
        new_obj = self.get_item_from_modulestore(chap_usage_key)
        self.assertEqual(new_obj.scope_ids.block_type, 'chapter')
        self.assertEqual(new_obj.display_name, display_name)
        self.assertEqual(new_obj.location.org, self.course.location.org)
        self.assertEqual(new_obj.location.course, self.course.location.course)

        # get the course and ensure it now points to this one
        course = self.get_item_from_modulestore(self.usage_key)
        self.assertIn(chap_usage_key, course.children)

        # use default display name
        resp = self.create_xblock(parent_usage_key=chap_usage_key, category='vertical')
        vert_usage_key = self.response_usage_key(resp)

        # create problem w/ boilerplate
        template_id = 'multiplechoice.yaml'
        resp = self.create_xblock(
            parent_usage_key=vert_usage_key,
            category='problem',
            boilerplate=template_id
        )
        prob_usage_key = self.response_usage_key(resp)
        problem = self.get_item_from_modulestore(prob_usage_key, True)
        # ensure it's draft
        self.assertTrue(problem.is_draft)
        # check against the template
        template = CapaDescriptor.get_template(template_id)
        self.assertEqual(problem.data, template['data'])
        self.assertEqual(problem.display_name, template['metadata']['display_name'])
        self.assertEqual(problem.markdown, template['metadata']['markdown'])

    def test_create_item_negative(self):
        """
        Negative tests for create_item
        """
        # non-existent boilerplate: creates a default
        resp = self.create_xblock(category='problem', boilerplate='nosuchboilerplate.yaml')
        self.assertEqual(resp.status_code, 200)

    def test_create_with_future_date(self):
        self.assertEqual(self.course.start, datetime(2030, 1, 1, tzinfo=UTC))
        resp = self.create_xblock(category='chapter')
        usage_key = self.response_usage_key(resp)
        obj = self.get_item_from_modulestore(usage_key)
        self.assertEqual(obj.start, datetime(2030, 1, 1, tzinfo=UTC))


class TestDuplicateItem(ItemTest):
    """
    Test the duplicate method.
    """
    def setUp(self):
        """ Creates the test course structure and a few components to 'duplicate'. """
        super(TestDuplicateItem, self).setUp()
        # Create a parent chapter (for testing children of children).
        resp = self.create_xblock(parent_usage_key=self.usage_key, category='chapter')
        self.chapter_usage_key = self.response_usage_key(resp)

        # create a sequential containing a problem and an html component
        resp = self.create_xblock(parent_usage_key=self.chapter_usage_key, category='sequential')
        self.seq_usage_key = self.response_usage_key(resp)

        # create problem and an html component
        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='problem', boilerplate='multiplechoice.yaml')
        self.problem_usage_key = self.response_usage_key(resp)

        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='html')
        self.html_usage_key = self.response_usage_key(resp)

        # Create a second sequential just (testing children of children)
        self.create_xblock(parent_usage_key=self.chapter_usage_key, category='sequential2')

    def test_duplicate_equality(self):
        """
        Tests that a duplicated xblock is identical to the original,
        except for location and display name.
        """
        def duplicate_and_verify(source_usage_key, parent_usage_key):
            usage_key = self._duplicate_item(parent_usage_key, source_usage_key)
            self.assertTrue(check_equality(source_usage_key, usage_key), "Duplicated item differs from original")

        def check_equality(source_usage_key, duplicate_usage_key):
            original_item = self.get_item_from_modulestore(source_usage_key, draft=True)
            duplicated_item = self.get_item_from_modulestore(duplicate_usage_key, draft=True)

            self.assertNotEqual(
                original_item.location,
                duplicated_item.location,
                "Location of duplicate should be different from original"
            )
            # Set the location and display name to be the same so we can make sure the rest of the duplicate is equal.
            duplicated_item.location = original_item.location
            duplicated_item.display_name = original_item.display_name

            # Children will also be duplicated, so for the purposes of testing equality, we will set
            # the children to the original after recursively checking the children.
            if original_item.has_children:
                self.assertEqual(
                    len(original_item.children),
                    len(duplicated_item.children),
                    "Duplicated item differs in number of children"
                )
                for i in xrange(len(original_item.children)):
                    if not check_equality(original_item.children[i], duplicated_item.children[i]):
                        return False
                duplicated_item.children = original_item.children

            return original_item == duplicated_item

        duplicate_and_verify(self.problem_usage_key, self.seq_usage_key)
        duplicate_and_verify(self.html_usage_key, self.seq_usage_key)
        duplicate_and_verify(self.seq_usage_key, self.chapter_usage_key)
        duplicate_and_verify(self.chapter_usage_key, self.usage_key)

    def test_ordering(self):
        """
        Tests the a duplicated xblock appears immediately after its source
        (if duplicate and source share the same parent), else at the
        end of the children of the parent.
        """
        def verify_order(source_usage_key, parent_usage_key, source_position=None):
            usage_key = self._duplicate_item(parent_usage_key, source_usage_key)
            parent = self.get_item_from_modulestore(parent_usage_key)
            children = parent.children
            if source_position is None:
                self.assertFalse(source_usage_key in children, 'source item not expected in children array')
                self.assertEqual(
                    children[len(children) - 1],
                    usage_key,
                    "duplicated item not at end"
                )
            else:
                self.assertEqual(
                    children[source_position],
                    source_usage_key,
                    "source item at wrong position"
                )
                self.assertEqual(
                    children[source_position + 1],
                    usage_key,
                    "duplicated item not ordered after source item"
                )

        verify_order(self.problem_usage_key, self.seq_usage_key, 0)
        # 2 because duplicate of problem should be located before.
        verify_order(self.html_usage_key, self.seq_usage_key, 2)
        verify_order(self.seq_usage_key, self.chapter_usage_key, 0)

        # Test duplicating something into a location that is not the parent of the original item.
        # Duplicated item should appear at the end.
        verify_order(self.html_usage_key, self.usage_key)

    def test_display_name(self):
        """
        Tests the expected display name for the duplicated xblock.
        """
        def verify_name(source_usage_key, parent_usage_key, expected_name, display_name=None):
            usage_key = self._duplicate_item(parent_usage_key, source_usage_key, display_name)
            duplicated_item = self.get_item_from_modulestore(usage_key, draft=True)
            self.assertEqual(duplicated_item.display_name, expected_name)
            return usage_key

        # Display name comes from template.
        dupe_usage_key = verify_name(self.problem_usage_key, self.seq_usage_key, "Duplicate of 'Multiple Choice'")
        # Test dupe of dupe.
        verify_name(dupe_usage_key, self.seq_usage_key, "Duplicate of 'Duplicate of 'Multiple Choice''")

        # Uses default display_name of 'Text' from HTML component.
        verify_name(self.html_usage_key, self.seq_usage_key, "Duplicate of 'Text'")

        # The sequence does not have a display_name set, so category is shown.
        verify_name(self.seq_usage_key, self.chapter_usage_key, "Duplicate of sequential")

        # Now send a custom display name for the duplicate.
        verify_name(self.seq_usage_key, self.chapter_usage_key, "customized name", display_name="customized name")

    def _duplicate_item(self, parent_usage_key, source_usage_key, display_name=None):
        data = {
            'parent_locator': unicode(parent_usage_key),
            'duplicate_source_locator': unicode(source_usage_key)
        }
        if display_name is not None:
            data['display_name'] = display_name

        resp = self.client.ajax_post(reverse('contentstore.views.xblock_handler'), json.dumps(data))
        return self.response_usage_key(resp)


class TestEditItem(ItemTest):
    """
    Test xblock update.
    """
    def setUp(self):
        """ Creates the test course structure and a couple problems to 'edit'. """
        super(TestEditItem, self).setUp()
        # create a chapter
        display_name = 'chapter created'
        resp = self.create_xblock(display_name=display_name, category='chapter')
        chap_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=chap_usage_key, category='sequential')
        self.seq_usage_key = self.response_usage_key(resp)
        self.seq_update_url = reverse_usage_url("xblock_handler", self.seq_usage_key)

        # create problem w/ boilerplate
        template_id = 'multiplechoice.yaml'
        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='problem', boilerplate=template_id)
        self.problem_usage_key = self.response_usage_key(resp)
        self.problem_update_url = reverse_usage_url("xblock_handler", self.problem_usage_key)

        self.course_update_url = reverse_usage_url("xblock_handler", self.usage_key)

    def test_delete_field(self):
        """
        Sending null in for a field 'deletes' it
        """
        self.client.ajax_post(
            self.problem_update_url,
            data={'metadata': {'rerandomize': 'onreset'}}
        )
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(problem.rerandomize, 'onreset')
        self.client.ajax_post(
            self.problem_update_url,
            data={'metadata': {'rerandomize': None}}
        )
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(problem.rerandomize, 'never')

    def test_null_field(self):
        """
        Sending null in for a field 'deletes' it
        """
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(problem.markdown)
        self.client.ajax_post(
            self.problem_update_url,
            data={'nullout': ['markdown']}
        )
        problem = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNone(problem.markdown)

    def test_date_fields(self):
        """
        Test setting due & start dates on sequential
        """
        sequential = self.get_item_from_modulestore(self.seq_usage_key)
        self.assertIsNone(sequential.due)
        self.client.ajax_post(
            self.seq_update_url,
            data={'metadata': {'due': '2010-11-22T04:00Z'}}
        )
        sequential = self.get_item_from_modulestore(self.seq_usage_key)
        self.assertEqual(sequential.due, datetime(2010, 11, 22, 4, 0, tzinfo=UTC))
        self.client.ajax_post(
            self.seq_update_url,
            data={'metadata': {'start': '2010-09-12T14:00Z'}}
        )
        sequential = self.get_item_from_modulestore(self.seq_usage_key)
        self.assertEqual(sequential.due, datetime(2010, 11, 22, 4, 0, tzinfo=UTC))
        self.assertEqual(sequential.start, datetime(2010, 9, 12, 14, 0, tzinfo=UTC))

    def test_delete_child(self):
        """
        Test deleting a child.
        """
        # Create 2 children of main course.
        resp_1 = self.create_xblock(display_name='child 1', category='chapter')
        resp_2 = self.create_xblock(display_name='child 2', category='chapter')
        chapter1_usage_key = self.response_usage_key(resp_1)
        chapter2_usage_key = self.response_usage_key(resp_2)

        course = self.get_item_from_modulestore(self.usage_key)
        self.assertIn(chapter1_usage_key, course.children)
        self.assertIn(chapter2_usage_key, course.children)

        # Remove one child from the course.
        resp = self.client.ajax_post(
            self.course_update_url,
            data={'children': [unicode(chapter2_usage_key)]}
        )
        self.assertEqual(resp.status_code, 200)

        # Verify that the child is removed.
        course = self.get_item_from_modulestore(self.usage_key)
        self.assertNotIn(chapter1_usage_key, course.children)
        self.assertIn(chapter2_usage_key, course.children)

    def test_reorder_children(self):
        """
        Test reordering children that can be in the draft store.
        """
        # Create 2 child units and re-order them. There was a bug about @draft getting added
        # to the IDs.
        unit_1_resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='vertical')
        unit_2_resp = self.create_xblock(parent_usage_key=self.seq_usage_key, category='vertical')
        unit1_usage_key = self.response_usage_key(unit_1_resp)
        unit2_usage_key = self.response_usage_key(unit_2_resp)

        # The sequential already has a child defined in the setUp (a problem).
        # Children must be on the sequential to reproduce the original bug,
        # as it is important that the parent (sequential) NOT be in the draft store.
        children = self.get_item_from_modulestore(self.seq_usage_key).children
        self.assertEqual(unit1_usage_key, children[1])
        self.assertEqual(unit2_usage_key, children[2])

        resp = self.client.ajax_post(
            self.seq_update_url,
            data={'children': [unicode(self.problem_usage_key), unicode(unit2_usage_key), unicode(unit1_usage_key)]}
        )
        self.assertEqual(resp.status_code, 200)

        children = self.get_item_from_modulestore(self.seq_usage_key).children
        self.assertEqual(self.problem_usage_key, children[0])
        self.assertEqual(unit1_usage_key, children[2])
        self.assertEqual(unit2_usage_key, children[1])

    def test_make_public(self):
        """ Test making a private problem public (publishing it). """
        # When the problem is first created, it is only in draft (because of its category).
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))

    def test_make_private(self):
        """ Test making a public problem private (un-publishing it). """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it private
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_private'}
        )
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)

    def test_make_draft(self):
        """ Test creating a draft version of a public problem. """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it draft, which means both versions will exist.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'create_draft'}
        )
        # Update the draft version and check that published is different.
        self.client.ajax_post(
            self.problem_update_url,
            data={'metadata': {'due': '2077-10-10T04:00Z'}}
        )
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        self.assertIsNone(published.due)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_make_public_with_update(self):
        """ Update a problem and make it public at the same time. """
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'metadata': {'due': '2077-10-10T04:00Z'},
                'publish': 'make_public'
            }
        )
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        self.assertEqual(published.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_make_private_with_update(self):
        """ Make a problem private and update it at the same time. """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'metadata': {'due': '2077-10-10T04:00Z'},
                'publish': 'make_private'
            }
        )
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_create_draft_with_update(self):
        """ Create a draft and update it at the same time. """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it draft, which means both versions will exist.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'metadata': {'due': '2077-10-10T04:00Z'},
                'publish': 'create_draft'
            }
        )
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        self.assertIsNone(published.due)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertEqual(draft.due, datetime(2077, 10, 10, 4, 0, tzinfo=UTC))

    def test_create_draft_with_multiple_requests(self):
        """
        Create a draft request returns already created version if it exists.
        """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        # Now make it draft, which means both versions will exist.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'create_draft'
            }
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))
        draft_1 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_1)

        # Now check that when a user sends request to create a draft when there is already a draft version then
        # user gets that already created draft instead of getting 'DuplicateItemError' exception.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'create_draft'
            }
        )
        draft_2 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_2)
        self.assertEqual(draft_1, draft_2)


    def test_make_private_with_multiple_requests(self):
        """
        Make private requests gets proper response even if xmodule is already made private.
        """
        # Make problem public.
        self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))

        # Now make it private, and check that its published version not exists
        resp = self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'make_private'
            }
        )
        self.assertEqual(resp.status_code, 200)
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        draft_1 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_1)

        # Now check that when a user sends request to make it private when it already is private then
        # user gets that private version instead of getting 'ItemNotFoundError' exception.
        self.client.ajax_post(
            self.problem_update_url,
            data={
                'publish': 'make_private'
            }
        )
        self.assertEqual(resp.status_code, 200)
        with self.assertRaises(ItemNotFoundError):
            self.get_item_from_modulestore(self.problem_usage_key, False)
        draft_2 = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertIsNotNone(draft_2)
        self.assertEqual(draft_1, draft_2)


    def test_published_and_draft_contents_with_update(self):
        """ Create a draft and publish it then modify the draft and check that published content is not modified """

        # Make problem public.
        resp = self.client.ajax_post(
            self.problem_update_url,
            data={'publish': 'make_public'}
        )
        self.assertIsNotNone(self.get_item_from_modulestore(self.problem_usage_key, False))

        # Now make a draft
        resp = self.client.ajax_post(
            self.problem_update_url,
            data={
                'id': unicode(self.problem_usage_key),
                'metadata': {},
                'data': "<p>Problem content draft.</p>",
                'publish': 'create_draft'
            }
        )

        # Both published and draft content should be different
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertNotEqual(draft.data, published.data)

        # Get problem by 'xblock_handler'
        view_url = reverse_usage_url("xblock_view_handler", self.problem_usage_key, {"view_name": "student_view"})
        resp = self.client.get(view_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)

        # Activate the editing view
        view_url = reverse_usage_url("xblock_view_handler", self.problem_usage_key, {"view_name": "studio_view"})
        resp = self.client.get(view_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)

        # Both published and draft content should still be different
        published = self.get_item_from_modulestore(self.problem_usage_key, False)
        draft = self.get_item_from_modulestore(self.problem_usage_key, True)
        self.assertNotEqual(draft.data, published.data)

    def test_publish_states_of_nested_xblocks(self):
        """ Test publishing of a unit page containing a nested xblock  """

        resp = self.create_xblock(parent_usage_key=self.seq_usage_key, display_name='Test Unit', category='vertical')
        unit_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=unit_usage_key, category='wrapper')
        wrapper_usage_key = self.response_usage_key(resp)
        resp = self.create_xblock(parent_usage_key=wrapper_usage_key, category='html')
        html_usage_key = self.response_usage_key(resp)

        # The unit and its children should be private initially
        unit_update_url = reverse_usage_url('xblock_handler', unit_usage_key)
        unit = self.get_item_from_modulestore(unit_usage_key, True)
        html = self.get_item_from_modulestore(html_usage_key, True)
        self.assertEqual(compute_publish_state(unit), PublishState.private)
        self.assertEqual(compute_publish_state(html), PublishState.private)

        # Make the unit public and verify that the problem is also made public
        resp = self.client.ajax_post(
            unit_update_url,
            data={'publish': 'make_public'}
        )
        self.assertEqual(resp.status_code, 200)
        unit = self.get_item_from_modulestore(unit_usage_key, True)
        html = self.get_item_from_modulestore(html_usage_key, True)
        self.assertEqual(compute_publish_state(unit), PublishState.public)
        self.assertEqual(compute_publish_state(html), PublishState.public)

        # Make a draft for the unit and verify that the problem also has a draft
        resp = self.client.ajax_post(
            unit_update_url,
            data={
                'id': unicode(unit_usage_key),
                'metadata': {},
                'publish': 'create_draft'
            }
        )
        self.assertEqual(resp.status_code, 200)
        unit = self.get_item_from_modulestore(unit_usage_key, True)
        html = self.get_item_from_modulestore(html_usage_key, True)
        self.assertEqual(compute_publish_state(unit), PublishState.draft)
        self.assertEqual(compute_publish_state(html), PublishState.draft)


@ddt.ddt
class TestComponentHandler(TestCase):
    def setUp(self):
        self.request_factory = RequestFactory()

        patcher = patch('contentstore.views.component.get_modulestore')
        self.get_modulestore = patcher.start()
        self.addCleanup(patcher.stop)

        self.descriptor = self.get_modulestore.return_value.get_item.return_value

        self.usage_key_string = unicode(
            Location('dummy_org', 'dummy_course', 'dummy_run', 'dummy_category', 'dummy_name')
        )

        self.user = UserFactory()

        self.request = self.request_factory.get('/dummy-url')
        self.request.user = self.user

    def test_invalid_handler(self):
        self.descriptor.handle.side_effect = Http404

        with self.assertRaises(Http404):
            component_handler(self.request, self.usage_key_string, 'invalid_handler')

    @ddt.data('GET', 'POST', 'PUT', 'DELETE')
    def test_request_method(self, method):

        def check_handler(handler, request, suffix):
            self.assertEquals(request.method, method)
            return Response()

        self.descriptor.handle = check_handler

        # Have to use the right method to create the request to get the HTTP method that we want
        req_factory_method = getattr(self.request_factory, method.lower())
        request = req_factory_method('/dummy-url')
        request.user = self.user

        component_handler(request, self.usage_key_string, 'dummy_handler')

    @ddt.data(200, 404, 500)
    def test_response_code(self, status_code):
        def create_response(handler, request, suffix):
            return Response(status_code=status_code)

        self.descriptor.handle = create_response

        self.assertEquals(component_handler(self.request, self.usage_key_string, 'dummy_handler').status_code, status_code)

########NEW FILE########
__FILENAME__ = test_preview
"""
Tests for contentstore.views.preview.py
"""
from django.test import TestCase
from django.test.client import RequestFactory

from student.tests.factories import UserFactory

from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory

from contentstore.views.preview import get_preview_fragment


class GetPreviewHtmlTestCase(TestCase):
    """
    Tests for get_preview_fragment.

    Note that there are other existing test cases in test_contentstore that indirectly execute
    get_preview_fragment via the xblock RESTful API.
    """

    def test_preview_fragment(self):
        """
        Test for calling get_preview_html.

        This test used to be specifically about Locators (ensuring that they did not
        get translated to Locations). The test now has questionable value.
        """
        course = CourseFactory.create()
        html = ItemFactory.create(
            parent_location=course.location,
            category="html",
            data={'data': "<html>foobar</html>"}
        )

        request = RequestFactory().get('/dummy-url')
        request.user = UserFactory()
        request.session = {}

        # Call get_preview_fragment directly.
        html = get_preview_fragment(request, html, {}).content

        # Verify student view html is returned, and the usage ID is as expected.
        self.assertRegexpMatches(
            html,
            'data-usage-id="location:MITx\+999\+Robot_Super_Course\+html\+html_[0-9]*"'
        )
        self.assertRegexpMatches(html, '<html>foobar</html>')

########NEW FILE########
__FILENAME__ = test_tabs
""" Tests for tab functions (just primitive). """

import json
from contentstore.views import tabs
from contentstore.tests.utils import CourseTestCase
from django.test import TestCase
from xmodule.modulestore.django import loc_mapper
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.tabs import CourseTabList, WikiTab
from contentstore.utils import reverse_course_url
from xmodule.modulestore.django import modulestore


class TabsPageTests(CourseTestCase):
    """Test cases for Tabs (a.k.a Pages) page"""

    def setUp(self):
        """Common setup for tests"""

        # call super class to setup course, etc.
        super(TabsPageTests, self).setUp()

        # Set the URL for tests
        self.url = reverse_course_url('tabs_handler', self.course.id)

        # add a static tab to the course, for code coverage
        self.test_tab = ItemFactory.create(
            parent_location=self.course.location,
            category="static_tab",
            display_name="Static_1"
        )
        self.reload_course()

    def check_invalid_tab_id_response(self, resp):
        """Verify response is an error listing the invalid_tab_id"""

        self.assertEqual(resp.status_code, 400)
        resp_content = json.loads(resp.content)
        self.assertIn("error", resp_content)
        self.assertIn("invalid_tab_id", resp_content['error'])

    def test_not_implemented(self):
        """Verify not implemented errors"""

        # JSON GET request not supported
        with self.assertRaises(NotImplementedError):
            self.client.get(self.url)

        # JSON POST request not supported
        with self.assertRaises(NotImplementedError):
            self.client.ajax_post(
                self.url,
                data=json.dumps({
                    'tab_id_locator': {'tab_id': WikiTab.type},
                    'unsupported_request': None,
                }),
            )

        # invalid JSON POST request
        with self.assertRaises(NotImplementedError):
            self.client.ajax_post(
                self.url,
                data={'invalid_request': None},
            )

    def test_view_index(self):
        """Basic check that the Pages page responds correctly"""

        resp = self.client.get_html(self.url)
        self.assertEqual(resp.status_code, 200)
        self.assertIn('course-nav-list', resp.content)

    def test_reorder_tabs(self):
        """Test re-ordering of tabs"""

        # get the original tab ids
        orig_tab_ids = [tab.tab_id for tab in self.course.tabs]
        tab_ids = list(orig_tab_ids)
        num_orig_tabs = len(orig_tab_ids)

        # make sure we have enough tabs to play around with
        self.assertTrue(num_orig_tabs >= 5)

        # reorder the last two tabs
        tab_ids[num_orig_tabs - 1], tab_ids[num_orig_tabs - 2] = tab_ids[num_orig_tabs - 2], tab_ids[num_orig_tabs - 1]

        # remove the middle tab
        # (the code needs to handle the case where tabs requested for re-ordering is a subset of the tabs in the course)
        removed_tab = tab_ids.pop(num_orig_tabs / 2)
        self.assertTrue(len(tab_ids) == num_orig_tabs - 1)

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data={'tabs': [{'tab_id': tab_id} for tab_id in tab_ids]},
        )
        self.assertEqual(resp.status_code, 204)

        # reload the course and verify the new tab order
        self.reload_course()
        new_tab_ids = [tab.tab_id for tab in self.course.tabs]
        self.assertEqual(new_tab_ids, tab_ids + [removed_tab])
        self.assertNotEqual(new_tab_ids, orig_tab_ids)

    def test_reorder_tabs_invalid_list(self):
        """Test re-ordering of tabs with invalid tab list"""

        orig_tab_ids = [tab.tab_id for tab in self.course.tabs]
        tab_ids = list(orig_tab_ids)

        # reorder the first two tabs
        tab_ids[0], tab_ids[1] = tab_ids[1], tab_ids[0]

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data={'tabs': [{'tab_id': tab_id} for tab_id in tab_ids]},
        )
        self.assertEqual(resp.status_code, 400)
        resp_content = json.loads(resp.content)
        self.assertIn("error", resp_content)

    def test_reorder_tabs_invalid_tab(self):
        """Test re-ordering of tabs with invalid tab"""

        invalid_tab_ids = ['courseware', 'info', 'invalid_tab_id']

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data={'tabs': [{'tab_id': tab_id} for tab_id in invalid_tab_ids]},
        )
        self.check_invalid_tab_id_response(resp)

    def check_toggle_tab_visiblity(self, tab_type, new_is_hidden_setting):
        """Helper method to check changes in tab visibility"""

        # find the tab
        old_tab = CourseTabList.get_tab_by_type(self.course.tabs, tab_type)

        # visibility should be different from new setting
        self.assertNotEqual(old_tab.is_hidden, new_is_hidden_setting)

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data=json.dumps({
                'tab_id_locator': {'tab_id': old_tab.tab_id},
                'is_hidden': new_is_hidden_setting,
            }),
        )
        self.assertEqual(resp.status_code, 204)

        # reload the course and verify the new visibility setting
        self.reload_course()
        new_tab = CourseTabList.get_tab_by_type(self.course.tabs, tab_type)
        self.assertEqual(new_tab.is_hidden, new_is_hidden_setting)

    def test_toggle_tab_visibility(self):
        """Test toggling of tab visiblity"""

        self.check_toggle_tab_visiblity(WikiTab.type, True)
        self.check_toggle_tab_visiblity(WikiTab.type, False)

    def test_toggle_invalid_tab_visibility(self):
        """Test toggling visibility of an invalid tab"""

        # post the request
        resp = self.client.ajax_post(
            self.url,
            data=json.dumps({
                'tab_id_locator': {'tab_id': 'invalid_tab_id'}
            }),
        )
        self.check_invalid_tab_id_response(resp)

    def test_tab_preview_html(self):
        """
        Verify that the static tab renders itself with the correct HTML
        """
        preview_url = '/xblock/{}/student_view'.format(self.test_tab.location)

        resp = self.client.get(preview_url, HTTP_ACCEPT='application/json')
        self.assertEqual(resp.status_code, 200)
        resp_content = json.loads(resp.content)
        html = resp_content['html']

        # Verify that the HTML contains the expected elements
        self.assertIn('<span class="action-button-text">Edit</span>', html)
        self.assertIn('<span class="sr">Duplicate this component</span>', html)
        self.assertIn('<span class="sr">Delete this component</span>', html)
        self.assertIn('<span data-tooltip="Drag to reorder" class="drag-handle action"></span>', html)



class PrimitiveTabEdit(TestCase):
    """Tests for the primitive tab edit data manipulations"""

    def test_delete(self):
        """Test primitive tab deletion."""
        course = CourseFactory.create(org='edX', course='999')
        with self.assertRaises(ValueError):
            tabs.primitive_delete(course, 0)
        with self.assertRaises(ValueError):
            tabs.primitive_delete(course, 1)
        with self.assertRaises(IndexError):
            tabs.primitive_delete(course, 6)
        tabs.primitive_delete(course, 2)
        self.assertFalse({u'type': u'textbooks'} in course.tabs)
        # Check that discussion has shifted up
        self.assertEquals(course.tabs[2], {'type': 'discussion', 'name': 'Discussion'})

    def test_insert(self):
        """Test primitive tab insertion."""
        course = CourseFactory.create(org='edX', course='999')
        tabs.primitive_insert(course, 2, 'notes', 'aname')
        self.assertEquals(course.tabs[2], {'type': 'notes', 'name': 'aname'})
        with self.assertRaises(ValueError):
            tabs.primitive_insert(course, 0, 'notes', 'aname')
        with self.assertRaises(ValueError):
            tabs.primitive_insert(course, 3, 'static_tab', 'aname')

    def test_save(self):
        """Test course saving."""
        course = CourseFactory.create(org='edX', course='999')
        tabs.primitive_insert(course, 3, 'notes', 'aname')
        course2 = modulestore().get_course(course.id)
        self.assertEquals(course2.tabs[3], {'type': 'notes', 'name': 'aname'})

########NEW FILE########
__FILENAME__ = test_textbooks
import json
from unittest import TestCase
from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url

from contentstore.views.course import (
    validate_textbooks_json, validate_textbook_json, TextbookValidationError)


class TextbookIndexTestCase(CourseTestCase):
    "Test cases for the textbook index page"
    def setUp(self):
        "Set the URL for tests"
        super(TextbookIndexTestCase, self).setUp()
        self.url = reverse_course_url('textbooks_list_handler', self.course.id)

    def test_view_index(self):
        "Basic check that the textbook index page responds correctly"
        resp = self.client.get(self.url)
        self.assertEqual(resp.status_code, 200)
        # we don't have resp.context right now,
        # due to bugs in our testing harness :(
        if resp.context:
            self.assertEqual(resp.context['course'], self.course)

    def test_view_index_xhr(self):
        "Check that we get a JSON response when requested via AJAX"
        resp = self.client.get(
            self.url,
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(self.course.pdf_textbooks, obj)

    def test_view_index_xhr_content(self):
        "Check that the response maps to the content of the modulestore"
        content = [
            {
                "tab_title": "my textbook",
                "url": "/abc.pdf",
                "id": "992"
            }, {
                "tab_title": "pineapple",
                "id": "0pineapple",
                "chapters": [
                    {
                        "title": "The Fruit",
                        "url": "/a/b/fruit.pdf",
                    }, {
                        "title": "The Legend",
                        "url": "/b/c/legend.pdf",
                    }
                ]
            }
        ]
        self.course.pdf_textbooks = content
        self.save_course()

        resp = self.client.get(
            self.url,
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 200)
        obj = json.loads(resp.content)
        self.assertEqual(content, obj)

    def test_view_index_xhr_put(self):
        "Check that you can save information to the server"
        textbooks = [
            {"tab_title": "Hi, mom!"},
            {"tab_title": "Textbook 2"},
        ]
        resp = self.client.put(
            self.url,
            data=json.dumps(textbooks),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 200)

        # should be the same, except for added ID
        no_ids = []
        self.reload_course()
        for textbook in self.course.pdf_textbooks:
            del textbook["id"]
            no_ids.append(textbook)
        self.assertEqual(no_ids, textbooks)

    def test_view_index_xhr_put_invalid(self):
        "Check that you can't save invalid JSON"
        resp = self.client.put(
            self.url,
            data="invalid",
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH='XMLHttpRequest'
        )
        self.assertEqual(resp.status_code, 400)
        obj = json.loads(resp.content)
        self.assertIn("error", obj)


class TextbookCreateTestCase(CourseTestCase):
    "Test cases for creating a new PDF textbook"

    def setUp(self):
        "Set up a url and some textbook content for tests"
        super(TextbookCreateTestCase, self).setUp()
        self.url = reverse_course_url('textbooks_list_handler', self.course.id)

        self.textbook = {
            "tab_title": "Economics",
            "chapters": {
                "title": "Chapter 1",
                "url": "/a/b/c/ch1.pdf",
            }
        }

    def test_happy_path(self):
        "Test that you can create a textbook"
        resp = self.client.post(
            self.url,
            data=json.dumps(self.textbook),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH="XMLHttpRequest",
        )
        self.assertEqual(resp.status_code, 201)
        self.assertIn("Location", resp)
        textbook = json.loads(resp.content)
        self.assertIn("id", textbook)
        del textbook["id"]
        self.assertEqual(self.textbook, textbook)

    def test_valid_id(self):
        "Textbook IDs must begin with a number; try a valid one"
        self.textbook["id"] = "7x5"
        resp = self.client.post(
            self.url,
            data=json.dumps(self.textbook),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH="XMLHttpRequest",
        )
        self.assertEqual(resp.status_code, 201)
        textbook = json.loads(resp.content)
        self.assertEqual(self.textbook, textbook)

    def test_invalid_id(self):
        "Textbook IDs must begin with a number; try an invalid one"
        self.textbook["id"] = "xxx"
        resp = self.client.post(
            self.url,
            data=json.dumps(self.textbook),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
            HTTP_X_REQUESTED_WITH="XMLHttpRequest",
        )
        self.assertEqual(resp.status_code, 400)
        self.assertNotIn("Location", resp)


class TextbookDetailTestCase(CourseTestCase):
    "Test cases for the `textbook_detail_handler` view"

    def setUp(self):
        "Set some useful content and URLs for tests"
        super(TextbookDetailTestCase, self).setUp()
        self.textbook1 = {
            "tab_title": "Economics",
            "id": 1,
            "chapters": {
                "title": "Chapter 1",
                "url": "/a/b/c/ch1.pdf",
            }
        }
        self.url1 = self.get_details_url("1")

        self.textbook2 = {
            "tab_title": "Algebra",
            "id": 2,
            "chapters": {
                "title": "Chapter 11",
                "url": "/a/b/ch11.pdf",
            }
        }
        self.url2 = self.get_details_url("2")
        self.course.pdf_textbooks = [self.textbook1, self.textbook2]
        # Save the data that we've just changed to the underlying
        # MongoKeyValueStore before we update the mongo datastore.
        self.save_course()
        self.url_nonexist = self.get_details_url("1=20")

    def get_details_url(self, textbook_id):
        """
        Returns the URL for textbook detail handler.
        """
        return reverse_course_url(
            'textbooks_detail_handler',
            self.course.id,
            kwargs={'textbook_id': textbook_id}
        )

    def test_get_1(self):
        "Get the first textbook"
        resp = self.client.get(self.url1)
        self.assertEqual(resp.status_code, 200)
        compare = json.loads(resp.content)
        self.assertEqual(compare, self.textbook1)

    def test_get_2(self):
        "Get the second textbook"
        resp = self.client.get(self.url2)
        self.assertEqual(resp.status_code, 200)
        compare = json.loads(resp.content)
        self.assertEqual(compare, self.textbook2)

    def test_get_nonexistant(self):
        "Get a nonexistent textbook"
        resp = self.client.get(self.url_nonexist)
        self.assertEqual(resp.status_code, 404)

    def test_delete(self):
        "Delete a textbook by ID"
        resp = self.client.delete(self.url1)
        self.assertEqual(resp.status_code, 204)
        self.reload_course()
        self.assertEqual(self.course.pdf_textbooks, [self.textbook2])

    def test_delete_nonexistant(self):
        "Delete a textbook by ID, when the ID doesn't match an existing textbook"
        resp = self.client.delete(self.url_nonexist)
        self.assertEqual(resp.status_code, 404)
        self.reload_course()
        self.assertEqual(self.course.pdf_textbooks, [self.textbook1, self.textbook2])

    def test_create_new_by_id(self):
        "Create a textbook by ID"
        textbook = {
            "tab_title": "a new textbook",
            "url": "supercool.pdf",
            "id": "1supercool",
        }
        url = self.get_details_url("1supercool")
        resp = self.client.post(
            url,
            data=json.dumps(textbook),
            content_type="application/json",
        )
        self.assertEqual(resp.status_code, 201)
        resp2 = self.client.get(url)
        self.assertEqual(resp2.status_code, 200)
        compare = json.loads(resp2.content)
        self.assertEqual(compare, textbook)
        self.reload_course()
        self.assertEqual(
            self.course.pdf_textbooks,
            [self.textbook1, self.textbook2, textbook]
        )

    def test_replace_by_id(self):
        "Create a textbook by ID, overwriting an existing textbook ID"
        replacement = {
            "tab_title": "You've been replaced!",
            "url": "supercool.pdf",
            "id": "2",
        }
        resp = self.client.post(
            self.url2,
            data=json.dumps(replacement),
            content_type="application/json",
        )
        self.assertEqual(resp.status_code, 201)
        resp2 = self.client.get(self.url2)
        self.assertEqual(resp2.status_code, 200)
        compare = json.loads(resp2.content)
        self.assertEqual(compare, replacement)
        course = self.store.get_item(self.course.location)
        self.assertEqual(
            course.pdf_textbooks,
            [self.textbook1, replacement]
        )


class TextbookValidationTestCase(TestCase):
    "Tests for the code to validate the structure of a PDF textbook"

    def setUp(self):
        "Set some useful content for tests"
        self.tb1 = {
            "tab_title": "Hi, mom!",
            "url": "/mom.pdf"
        }
        self.tb2 = {
            "tab_title": "Hi, dad!",
            "chapters": [
                {
                    "title": "Baseball",
                    "url": "baseball.pdf",
                }, {
                    "title": "Basketball",
                    "url": "crazypants.pdf",
                }
            ]
        }
        self.textbooks = [self.tb1, self.tb2]

    def test_happy_path_plural(self):
        "Test that the plural validator works properly"
        result = validate_textbooks_json(json.dumps(self.textbooks))
        self.assertEqual(self.textbooks, result)

    def test_happy_path_singular_1(self):
        "Test that the singular validator works properly"
        result = validate_textbook_json(json.dumps(self.tb1))
        self.assertEqual(self.tb1, result)

    def test_happy_path_singular_2(self):
        "Test that the singular validator works properly, with different data"
        result = validate_textbook_json(json.dumps(self.tb2))
        self.assertEqual(self.tb2, result)

    def test_valid_id(self):
        "Test that a valid ID doesn't trip the validator, and comes out unchanged"
        self.tb1["id"] = 1
        result = validate_textbook_json(json.dumps(self.tb1))
        self.assertEqual(self.tb1, result)

    def test_invalid_id(self):
        "Test that an invalid ID trips the validator"
        self.tb1["id"] = "abc"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json(json.dumps(self.tb1))

    def test_invalid_json_plural(self):
        "Test that invalid JSON trips the plural validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json("[{'abc'}]")

    def test_invalid_json_singular(self):
        "Test that invalid JSON trips the singluar validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json("[{1]}")

    def test_wrong_json_plural(self):
        "Test that a JSON object trips the plural validators (requires a list)"
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json('{"tab_title": "Hi, mom!"}')

    def test_wrong_json_singular(self):
        "Test that a JSON list trips the plural validators (requires an object)"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json('[{"tab_title": "Hi, mom!"}, {"tab_title": "Hi, dad!"}]')

    def test_no_tab_title_plural(self):
        "Test that `tab_title` is required for the plural validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json('[{"url": "/textbook.pdf"}]')

    def test_no_tab_title_singular(self):
        "Test that `tab_title` is required for the singular validator"
        with self.assertRaises(TextbookValidationError):
            validate_textbook_json('{"url": "/textbook.pdf"}')

    def test_duplicate_ids(self):
        "Test that duplicate IDs in the plural validator trips the validator"
        textbooks = [{
            "tab_title": "name one",
            "url": "one.pdf",
            "id": 1,
        }, {
            "tab_title": "name two",
            "url": "two.pdf",
            "id": 1,
        }]
        with self.assertRaises(TextbookValidationError):
            validate_textbooks_json(json.dumps(textbooks))

########NEW FILE########
__FILENAME__ = test_transcripts
"""Tests for items views."""

import os
import json
import tempfile
from uuid import uuid4
import copy
import textwrap
from pymongo import MongoClient

from django.core.urlresolvers import reverse
from django.test.utils import override_settings
from django.conf import settings

from contentstore.tests.utils import CourseTestCase
from cache_toolbox.core import del_cached_content
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.keys import UsageKey
from xmodule.video_module import transcripts_utils

from contentstore.tests.modulestore_config import TEST_MODULESTORE
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class Basetranscripts(CourseTestCase):
    """Base test class for transcripts tests."""

    def clear_subs_content(self):
        """Remove, if transcripts content exists."""
        for youtube_id in self.get_youtube_ids().values():
            filename = 'subs_{0}.srt.sjson'.format(youtube_id)
            content_location = StaticContent.compute_location(self.course.id, filename)
            try:
                content = contentstore().find(content_location)
                contentstore().delete(content.get_id())
            except NotFoundError:
                pass

    def setUp(self):
        """Create initial data."""
        super(Basetranscripts, self).setUp()

        # Add video module
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'video',
            'type': 'video'
        }
        resp = self.client.ajax_post('/xblock/', data)
        self.assertEqual(resp.status_code, 200)

        self.video_usage_key = self._get_usage_key(resp)
        self.item = modulestore().get_item(self.video_usage_key)
        # hI10vDNYz4M - valid Youtube ID with transcripts.
        # JMD_ifUUfsU, AKqURZnYqpk, DYpADpL7jAY - valid Youtube IDs without transcripts.
        self.item.data = '<video youtube="0.75:JMD_ifUUfsU,1.0:hI10vDNYz4M,1.25:AKqURZnYqpk,1.50:DYpADpL7jAY" />'
        modulestore().update_item(self.item, self.user.id)

        self.item = modulestore().get_item(self.video_usage_key)
        # Remove all transcripts for current module.
        self.clear_subs_content()

    def _get_usage_key(self, resp):
        """ Returns the usage key from the response returned by a create operation. """
        usage_key_string = json.loads(resp.content).get('locator')
        return UsageKey.from_string(usage_key_string)

    def get_youtube_ids(self):
        """Return youtube speeds and ids."""
        item = modulestore().get_item(self.video_usage_key)

        return {
            0.75: item.youtube_id_0_75,
            1: item.youtube_id_1_0,
            1.25: item.youtube_id_1_25,
            1.5: item.youtube_id_1_5
        }

    def tearDown(self):
        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()


class TestUploadtranscripts(Basetranscripts):
    """Tests for '/transcripts/upload' url."""

    def setUp(self):
        """Create initial data."""
        super(TestUploadtranscripts, self).setUp()

        self.good_srt_file = tempfile.NamedTemporaryFile(suffix='.srt')
        self.good_srt_file.write(textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """))
        self.good_srt_file.seek(0)

        self.bad_data_srt_file = tempfile.NamedTemporaryFile(suffix='.srt')
        self.bad_data_srt_file.write('Some BAD data')
        self.bad_data_srt_file.seek(0)

        self.bad_name_srt_file = tempfile.NamedTemporaryFile(suffix='.BAD')
        self.bad_name_srt_file.write(textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """))
        self.bad_name_srt_file.seek(0)

    def test_success_video_module_source_subs_uploading(self):
        self.item.data = textwrap.dedent("""
            <video youtube="">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """)
        modulestore().update_item(self.item, self.user.id)

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': self.video_usage_key,
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(json.loads(resp.content).get('status'), 'Success')

        item = modulestore().get_item(self.video_usage_key)
        self.assertEqual(item.sub, filename)

        content_location = StaticContent.compute_location(
            self.course.id, 'subs_{0}.srt.sjson'.format(filename))
        self.assertTrue(contentstore().find(content_location))

    def test_fail_data_without_id(self):
        link = reverse('upload_transcripts')
        resp = self.client.post(link, {'transcript-file': self.good_srt_file})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'POST data without "locator" form data.')

    def test_fail_data_without_file(self):
        link = reverse('upload_transcripts')
        resp = self.client.post(link, {'locator': self.video_usage_key})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'POST data without "file" form data.')

    def test_fail_data_with_bad_locator(self):
        # Test for raising `InvalidLocationError` exception.
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': 'BAD_LOCATOR',
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

        # Test for raising `ItemNotFoundError` exception.
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': '{0}_{1}'.format(self.video_usage_key, 'BAD_LOCATOR'),
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

    def test_fail_for_non_video_module(self):
        # non_video module: setup
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'non_video',
            'type': 'non_video'
        }
        resp = self.client.ajax_post('/xblock/', data)
        usage_key = self._get_usage_key(resp)
        item = modulestore().get_item(usage_key)
        item.data = '<non_video youtube="0.75:JMD_ifUUfsU,1.0:hI10vDNYz4M" />'
        modulestore().update_item(item, self.user.id)

        # non_video module: testing

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(usage_key),
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Transcripts are supported only for "video" modules.')

    def test_fail_bad_xml(self):
        self.item.data = '<<<video youtube="0.75:JMD_ifUUfsU,1.25:AKqURZnYqpk,1.50:DYpADpL7jAY" />'
        modulestore().update_item(self.item, self.user.id)

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.good_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(self.video_usage_key),
            'transcript-file': self.good_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })

        self.assertEqual(resp.status_code, 400)
        # incorrect xml produces incorrect item category error
        self.assertEqual(json.loads(resp.content).get('status'), 'Transcripts are supported only for "video" modules.')

    def test_fail_bad_data_srt_file(self):
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.bad_data_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(self.video_usage_key),
            'transcript-file': self.bad_data_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Something wrong with SubRip transcripts file during parsing.')

    def test_fail_bad_name_srt_file(self):
        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(self.bad_name_srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': unicode(self.video_usage_key),
            'transcript-file': self.bad_name_srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'We support only SubRip (*.srt) transcripts format.')

    def test_undefined_file_extension(self):
        srt_file = tempfile.NamedTemporaryFile(suffix='')
        srt_file.write(textwrap.dedent("""
            1
            00:00:10,500 --> 00:00:13,000
            Elephant's Dream

            2
            00:00:15,000 --> 00:00:18,000
            At the left we can see...
        """))
        srt_file.seek(0)

        link = reverse('upload_transcripts')
        filename = os.path.splitext(os.path.basename(srt_file.name))[0]
        resp = self.client.post(link, {
            'locator': self.video_usage_key,
            'transcript-file': srt_file,
            'video_list': json.dumps([{
                'type': 'html5',
                'video': filename,
                'mode': 'mp4',
            }])
        })
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Undefined file extension.')

    def tearDown(self):
        super(TestUploadtranscripts, self).tearDown()

        self.good_srt_file.close()
        self.bad_data_srt_file.close()
        self.bad_name_srt_file.close()


class TestDownloadtranscripts(Basetranscripts):
    """Tests for '/transcripts/download' url."""

    def save_subs_to_store(self, subs, subs_id):
        """Save transcripts into `StaticContent`."""
        filedata = json.dumps(subs, indent=2)
        mime_type = 'application/json'
        filename = 'subs_{0}.srt.sjson'.format(subs_id)

        content_location = StaticContent.compute_location(self.course.id, filename)
        content = StaticContent(content_location, filename, mime_type, filedata)
        contentstore().save(content)
        del_cached_content(content_location)
        return content_location

    def test_success_download_youtube(self):
        self.item.data = '<video youtube="1:JMD_ifUUfsU" />'
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, 'JMD_ifUUfsU')

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key, 'subs_id': "JMD_ifUUfsU"})
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(resp.content, """0\n00:00:00,100 --> 00:00:00,200\nsubs #1\n\n1\n00:00:00,200 --> 00:00:00,240\nsubs #2\n\n2\n00:00:00,240 --> 00:00:00,380\nsubs #3\n\n""")

    def test_success_download_nonyoutube(self):
        subs_id = str(uuid4())
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """.format(subs_id))
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key, 'subs_id': subs_id})
        self.assertEqual(resp.status_code, 200)
        self.assertEqual(
            resp.content,
            '0\n00:00:00,100 --> 00:00:00,200\nsubs #1\n\n1\n00:00:00,200 --> '
            '00:00:00,240\nsubs #2\n\n2\n00:00:00,240 --> 00:00:00,380\nsubs #3\n\n'
        )
        transcripts_utils.remove_subs_from_store(subs_id, self.item)

    def test_fail_data_without_file(self):
        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': ''})
        self.assertEqual(resp.status_code, 404)

        resp = self.client.get(link, {})
        self.assertEqual(resp.status_code, 404)

    def test_fail_data_with_bad_locator(self):
        # Test for raising `InvalidLocationError` exception.
        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': 'BAD_LOCATOR'})
        self.assertEqual(resp.status_code, 404)

        # Test for raising `ItemNotFoundError` exception.
        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': '{0}_{1}'.format(self.video_usage_key, 'BAD_LOCATOR')})
        self.assertEqual(resp.status_code, 404)

    def test_fail_for_non_video_module(self):
        # Video module: setup
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'videoalpha',
            'type': 'videoalpha'
        }
        resp = self.client.ajax_post('/xblock/', data)
        usage_key = self._get_usage_key(resp)
        subs_id = str(uuid4())
        item = modulestore().get_item(usage_key)
        item.data = textwrap.dedent("""
            <videoalpha youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </videoalpha>
        """.format(subs_id))
        modulestore().update_item(item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': unicode(usage_key)})
        self.assertEqual(resp.status_code, 404)

    def test_fail_nonyoutube_subs_dont_exist(self):
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="UNDEFINED">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """)
        modulestore().update_item(self.item, self.user.id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key})
        self.assertEqual(resp.status_code, 404)

    def test_empty_youtube_attr_and_sub_attr(self):
        self.item.data = textwrap.dedent("""
            <video youtube="">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """)
        modulestore().update_item(self.item, self.user.id)

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key})

        self.assertEqual(resp.status_code, 404)

    def test_fail_bad_sjson_subs(self):
        subs_id = str(uuid4())
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """.format(subs_id))
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1'
            ]
        }
        self.save_subs_to_store(subs, 'JMD_ifUUfsU')

        link = reverse('download_transcripts')
        resp = self.client.get(link, {'locator': self.video_usage_key})

        self.assertEqual(resp.status_code, 404)


class TestChecktranscripts(Basetranscripts):
    """Tests for '/transcripts/check' url."""

    def save_subs_to_store(self, subs, subs_id):
        """Save transcripts into `StaticContent`."""
        filedata = json.dumps(subs, indent=2)
        mime_type = 'application/json'
        filename = 'subs_{0}.srt.sjson'.format(subs_id)

        content_location = StaticContent.compute_location(self.course.id, filename)
        content = StaticContent(content_location, filename, mime_type, filedata)
        contentstore().save(content)
        del_cached_content(content_location)
        return content_location

    def test_success_download_nonyoutube(self):
        subs_id = str(uuid4())
        self.item.data = textwrap.dedent("""
            <video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </video>
        """.format(subs_id))
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        data = {
            'locator': unicode(self.video_usage_key),
            'videos': [{
                'type': 'html5',
                'video': subs_id,
                'mode': 'mp4',
            }]
        }
        link = reverse('check_transcripts')
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 200)
        self.assertDictEqual(
            json.loads(resp.content),
            {
                u'status': u'Success',
                u'subs': unicode(subs_id),
                u'youtube_local': False,
                u'is_youtube_mode': False,
                u'youtube_server': False,
                u'command': u'found',
                u'current_item_subs': unicode(subs_id),
                u'youtube_diff': True,
                u'html5_local': [unicode(subs_id)],
                u'html5_equal': False,
            }
        )

        transcripts_utils.remove_subs_from_store(subs_id, self.item)

    def test_check_youtube(self):
        self.item.data = '<video youtube="1:JMD_ifUUfsU" />'
        modulestore().update_item(self.item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, 'JMD_ifUUfsU')
        link = reverse('check_transcripts')
        data = {
            'locator': unicode(self.video_usage_key),
            'videos': [{
                'type': 'youtube',
                'video': 'JMD_ifUUfsU',
                'mode': 'youtube',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 200)
        self.assertDictEqual(
            json.loads(resp.content),
            {
                u'status': u'Success',
                u'subs': u'JMD_ifUUfsU',
                u'youtube_local': True,
                u'is_youtube_mode': True,
                u'youtube_server': False,
                u'command': u'found',
                u'current_item_subs': None,
                u'youtube_diff': True,
                u'html5_local': [],
                u'html5_equal': False,
            }
        )

    def test_fail_data_without_id(self):
        link = reverse('check_transcripts')
        data = {
            'locator': '',
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

    def test_fail_data_with_bad_locator(self):
        # Test for raising `InvalidLocationError` exception.
        link = reverse('check_transcripts')
        data = {
            'locator': '',
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

        # Test for raising `ItemNotFoundError` exception.
        data = {
            'locator': '{0}_{1}'.format(self.video_usage_key, 'BAD_LOCATOR'),
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), "Can't find item by locator.")

    def test_fail_for_non_video_module(self):
        # Not video module: setup
        data = {
            'parent_locator': unicode(self.course.location),
            'category': 'not_video',
            'type': 'not_video'
        }
        resp = self.client.ajax_post('/xblock/', data)
        usage_key = self._get_usage_key(resp)
        subs_id = str(uuid4())
        item = modulestore().get_item(usage_key)
        item.data = textwrap.dedent("""
            <not_video youtube="" sub="{}">
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.mp4"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.webm"/>
                <source src="http://www.quirksmode.org/html5/videos/big_buck_bunny.ogv"/>
            </videoalpha>
        """.format(subs_id))
        modulestore().update_item(item, self.user.id)

        subs = {
            'start': [100, 200, 240],
            'end': [200, 240, 380],
            'text': [
                'subs #1',
                'subs #2',
                'subs #3'
            ]
        }
        self.save_subs_to_store(subs, subs_id)

        data = {
            'locator': unicode(usage_key),
            'videos': [{
                'type': '',
                'video': '',
                'mode': '',
            }]
        }
        link = reverse('check_transcripts')
        resp = self.client.get(link, {'data': json.dumps(data)})
        self.assertEqual(resp.status_code, 400)
        self.assertEqual(json.loads(resp.content).get('status'), 'Transcripts are supported only for "video" modules.')

########NEW FILE########
__FILENAME__ = test_unit_page
"""
Unit tests for the unit page.
"""

from contentstore.views.tests.utils import StudioPageTestCase
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import ItemFactory


class UnitPageTestCase(StudioPageTestCase):
    """
    Unit tests for the unit page.
    """

    def setUp(self):
        super(UnitPageTestCase, self).setUp()
        self.vertical = ItemFactory.create(parent_location=self.sequential.location,
                                           category='vertical', display_name='Unit')
        self.video = ItemFactory.create(parent_location=self.vertical.location,
                                        category="video", display_name="My Video")

    def test_public_unit_page_html(self):
        """
        Verify that an xblock returns the expected HTML for a public unit page.
        """
        html = self.get_page_html(self.vertical)
        self.validate_html_for_add_buttons(html)

    def test_draft_unit_page_html(self):
        """
        Verify that an xblock returns the expected HTML for a draft unit page.
        """
        draft_unit = modulestore('draft').convert_to_draft(self.vertical.location)
        html = self.get_page_html(draft_unit)
        self.validate_html_for_add_buttons(html)

    def test_public_component_preview_html(self):
        """
        Verify that a public xblock's preview returns the expected HTML.
        """
        self.validate_preview_html(self.video, 'student_view',
                                   can_edit=True, can_reorder=True, can_add=False)

    def test_draft_component_preview_html(self):
        """
        Verify that a draft xblock's preview returns the expected HTML.
        """
        modulestore('draft').convert_to_draft(self.vertical.location)
        draft_video = modulestore('draft').convert_to_draft(self.video.location)
        self.validate_preview_html(draft_video, 'student_view',
                                   can_edit=True, can_reorder=True, can_add=False)

    def test_public_child_container_preview_html(self):
        """
        Verify that a public child container rendering on the unit page (which shows a View arrow
        to the container page) returns the expected HTML.
        """
        child_container = ItemFactory.create(parent_location=self.vertical.location,
                                             category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=child_container.location,
                           category='html', display_name='grandchild')
        self.validate_preview_html(child_container, 'student_view',
                                   can_reorder=True, can_edit=False, can_add=False)

    def test_draft_child_container_preview_html(self):
        """
        Verify that a draft child container rendering on the unit page (which shows a View arrow
        to the container page) returns the expected HTML.
        """
        child_container = ItemFactory.create(parent_location=self.vertical.location,
                                             category='split_test', display_name='Split Test')
        ItemFactory.create(parent_location=child_container.location,
                           category='html', display_name='grandchild')
        modulestore('draft').convert_to_draft(self.vertical.location)
        draft_child_container = modulestore('draft').get_item(child_container.location)
        self.validate_preview_html(draft_child_container, 'student_view',
                                   can_reorder=True, can_edit=False, can_add=False)

########NEW FILE########
__FILENAME__ = test_user
"""
Tests for contentstore/views/user.py.
"""
import json

from contentstore.tests.utils import CourseTestCase
from contentstore.utils import reverse_course_url
from django.contrib.auth.models import User
from student.models import CourseEnrollment
from student.roles import CourseStaffRole, CourseInstructorRole
from student import auth


class UsersTestCase(CourseTestCase):
    def setUp(self):
        super(UsersTestCase, self).setUp()
        self.ext_user = User.objects.create_user(
            "joe", "joe@comedycentral.com", "haha")
        self.ext_user.is_active = True
        self.ext_user.is_staff = False
        self.ext_user.save()
        self.inactive_user = User.objects.create_user(
            "carl", "carl@comedycentral.com", "haha")
        self.inactive_user.is_active = False
        self.inactive_user.is_staff = False
        self.inactive_user.save()

        self.index_url = self.course_team_url()
        self.detail_url = self.course_team_url(email=self.ext_user.email)
        self.inactive_detail_url = self.course_team_url(email=self.inactive_user.email)
        self.invalid_detail_url = self.course_team_url(email='nonexistent@user.com')

    def course_team_url(self, email=None):
        return reverse_course_url(
            'course_team_handler', self.course.id,
            kwargs={'email': email} if email else {}
        )

    def test_index(self):
        resp = self.client.get(self.index_url, HTTP_ACCEPT='text/html')
        # ext_user is not currently a member of the course team, and so should
        # not show up on the page.
        self.assertNotContains(resp, self.ext_user.email)

    def test_index_member(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.ext_user)

        resp = self.client.get(self.index_url, HTTP_ACCEPT='text/html')
        self.assertContains(resp, self.ext_user.email)

    def test_detail(self):
        resp = self.client.get(self.detail_url)
        self.assertEqual(resp.status_code, 200)
        result = json.loads(resp.content)
        self.assertEqual(result["role"], None)
        self.assertTrue(result["active"])

    def test_detail_inactive(self):
        resp = self.client.get(self.inactive_detail_url)
        self.assertEqual(resp.status_code, 200)
        result = json.loads(resp.content)
        self.assertFalse(result["active"])

    def test_detail_invalid(self):
        resp = self.client.get(self.invalid_detail_url)
        self.assertEqual(resp.status_code, 404)
        result = json.loads(resp.content)
        self.assertIn("error", result)

    def test_detail_post(self):
        resp = self.client.post(
            self.detail_url,
            data={"role": None},
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        # no content: should not be in any roles
        self.assertFalse(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_not_enrolled()

    def test_detail_post_staff(self):
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_enrolled()

    def test_detail_post_staff_other_inst(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.user)

        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_enrolled()
        # check that other user is unchanged
        user = User.objects.get(email=self.user.email)
        self.assertTrue(auth.has_access(user, CourseInstructorRole(self.course.id)))
        self.assertFalse(CourseStaffRole(self.course.id).has_user(user))

    def test_detail_post_instructor(self):
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "instructor"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assertFalse(CourseStaffRole(self.course.id).has_user(ext_user))
        self.assert_enrolled()

    def test_detail_post_missing_role(self):
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"toys": "fun"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        self.assert_not_enrolled()

    def test_detail_post_no_json(self):
        resp = self.client.post(
            self.detail_url,
            data={"role": "staff"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))
        self.assert_enrolled()

    def test_detail_delete_staff(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.ext_user)

        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertFalse(auth.has_access(ext_user, CourseStaffRole(self.course.id)))

    def test_detail_delete_instructor(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.ext_user, self.user)

        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertFalse(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))

    def test_delete_last_instructor(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.ext_user)

        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))

    def test_post_last_instructor(self):
        auth.add_users(self.user, CourseInstructorRole(self.course.id), self.ext_user)

        resp = self.client.post(
            self.detail_url,
            data={"role": "staff"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseInstructorRole(self.course.id)))

    def test_permission_denied_self(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user)
        self.user.is_staff = False
        self.user.save()

        self_url = self.course_team_url(email=self.user.email)

        resp = self.client.post(
            self_url,
            data={"role": "instructor"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)

    def test_permission_denied_other(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user)
        self.user.is_staff = False
        self.user.save()

        resp = self.client.post(
            self.detail_url,
            data={"role": "instructor"},
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)

    def test_staff_can_delete_self(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user)
        self.user.is_staff = False
        self.user.save()

        self_url = self.course_team_url(email=self.user.email)

        resp = self.client.delete(self_url)
        self.assertEqual(resp.status_code, 204)
        # reload user from DB
        user = User.objects.get(email=self.user.email)
        self.assertFalse(auth.has_access(user, CourseStaffRole(self.course.id)))

    def test_staff_cannot_delete_other(self):
        auth.add_users(self.user, CourseStaffRole(self.course.id), self.user, self.ext_user)
        self.user.is_staff = False
        self.user.save()

        resp = self.client.delete(self.detail_url)
        self.assertEqual(resp.status_code, 400)
        result = json.loads(resp.content)
        self.assertIn("error", result)
        # reload user from DB
        ext_user = User.objects.get(email=self.ext_user.email)
        self.assertTrue(auth.has_access(ext_user, CourseStaffRole(self.course.id)))

    def test_user_not_initially_enrolled(self):
        # Verify that ext_user is not enrolled in the new course before being added as a staff member.
        self.assert_not_enrolled()

    def test_remove_staff_does_not_unenroll(self):
        # Add user with staff permissions.
        self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assert_enrolled()
        # Remove user from staff on course. Will not un-enroll them from the course.
        resp = self.client.delete(
            self.detail_url,
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        self.assert_enrolled()

    def test_staff_to_instructor_still_enrolled(self):
        # Add user with staff permission.
        self.client.post(
            self.detail_url,
            data=json.dumps({"role": "staff"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assert_enrolled()
        # Now add with instructor permission. Verify still enrolled.
        resp = self.client.post(
            self.detail_url,
            data=json.dumps({"role": "instructor"}),
            content_type="application/json",
            HTTP_ACCEPT="application/json",
        )
        self.assertEqual(resp.status_code, 204)
        self.assert_enrolled()

    def assert_not_enrolled(self):
        """ Asserts that self.ext_user is not enrolled in self.course. """
        self.assertFalse(
            CourseEnrollment.is_enrolled(self.ext_user, self.course.id),
            'Did not expect ext_user to be enrolled in course'
        )

    def assert_enrolled(self):
        """ Asserts that self.ext_user is enrolled in self.course. """
        self.assertTrue(
            CourseEnrollment.is_enrolled(self.ext_user, self.course.id),
            'User ext_user should have been enrolled in the course'
        )

########NEW FILE########
__FILENAME__ = utils
"""
Utilities for view tests.
"""

import json

from contentstore.tests.utils import CourseTestCase
from contentstore.views.helpers import xblock_studio_url
from xmodule.modulestore.tests.factories import ItemFactory


class StudioPageTestCase(CourseTestCase):
    """
    Base class for all tests of Studio pages.
    """

    def setUp(self):
        super(StudioPageTestCase, self).setUp()
        self.chapter = ItemFactory.create(parent_location=self.course.location,
                                          category='chapter', display_name="Week 1")
        self.sequential = ItemFactory.create(parent_location=self.chapter.location,
                                             category='sequential', display_name="Lesson 1")

    def get_page_html(self, xblock):
        """
        Returns the HTML for the page representing the xblock.
        """
        url = xblock_studio_url(xblock)
        self.assertIsNotNone(url)
        resp = self.client.get_html(url)
        self.assertEqual(resp.status_code, 200)
        return resp.content

    def get_preview_html(self, xblock, view_name):
        """
        Returns the HTML for the xblock when shown within a unit or container page.
        """
        preview_url = '/xblock/{usage_key}/{view_name}'.format(usage_key=xblock.location, view_name=view_name)
        resp = self.client.get_json(preview_url)
        self.assertEqual(resp.status_code, 200)
        resp_content = json.loads(resp.content)
        return resp_content['html']

    def validate_preview_html(self, xblock, view_name, can_edit=True, can_reorder=True, can_add=True):
        """
        Verify that the specified xblock's preview has the expected HTML elements.
        """
        html = self.get_preview_html(xblock, view_name)
        self.validate_html_for_add_buttons(html, can_add=can_add)

        # Verify that there are no drag handles for public blocks
        drag_handle_html = '<span data-tooltip="Drag to reorder" class="drag-handle action"></span>'
        if can_reorder:
            self.assertIn(drag_handle_html, html)
        else:
            self.assertNotIn(drag_handle_html, html)

        # Verify that there are no action buttons for public blocks
        expected_button_html = [
            '<a href="#" class="edit-button action-button">',
            '<a href="#" data-tooltip="Delete" class="delete-button action-button">',
            '<a href="#" data-tooltip="Duplicate" class="duplicate-button action-button">'
        ]
        for button_html in expected_button_html:
            if can_edit:
                self.assertIn(button_html, html)
            else:
                self.assertNotIn(button_html, html)

    def validate_html_for_add_buttons(self, html, can_add=True):
        """
        Validate that the specified HTML has the appropriate add actions for the current publish state.
        """
        # Verify that there are no add buttons for public blocks
        add_button_html = '<div class="add-xblock-component new-component-item adding"></div>'
        if can_add:
            self.assertIn(add_button_html, html)
        else:
            self.assertNotIn(add_button_html, html)

########NEW FILE########
__FILENAME__ = transcripts_ajax
"""
Actions manager for transcripts ajax calls.
+++++++++++++++++++++++++++++++++++++++++++

Module do not support rollback (pressing "Cancel" button in Studio)
All user changes are saved immediately.
"""
import copy
import os
import logging
import json
import requests

from django.http import HttpResponse, Http404
from django.core.exceptions import PermissionDenied
from django.contrib.auth.decorators import login_required
from django.conf import settings
from django.utils.translation import ugettext as _

from opaque_keys import InvalidKeyError

from xmodule.contentstore.content import StaticContent
from xmodule.exceptions import NotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import UsageKey
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.exceptions import ItemNotFoundError

from util.json_request import JsonResponse

from xmodule.video_module.transcripts_utils import (
    generate_subs_from_source,
    generate_srt_from_sjson, remove_subs_from_store,
    download_youtube_subs, get_transcripts_from_youtube,
    copy_or_rename_transcript,
    manage_video_subtitles_save,
    GetTranscriptsFromYouTubeException,
    TranscriptsRequestValidationException
)

from .access import has_course_access

__all__ = [
    'upload_transcripts',
    'download_transcripts',
    'check_transcripts',
    'choose_transcripts',
    'replace_transcripts',
    'rename_transcripts',
    'save_transcripts',
]

log = logging.getLogger(__name__)


def error_response(response, message, status_code=400):
    """
    Simplify similar actions: log message and return JsonResponse with message included in response.

    By default return 400 (Bad Request) Response.
    """
    log.debug(message)
    response['status'] = message
    return JsonResponse(response, status_code)


@login_required
def upload_transcripts(request):
    """
    Upload transcripts for current module.

    returns: response dict::

        status: 'Success' and HTTP 200 or 'Error' and HTTP 400.
        subs: Value of uploaded and saved html5 sub field in video item.
    """
    response = {
        'status': 'Unknown server error',
        'subs': '',
    }

    locator = request.POST.get('locator')
    if not locator:
        return error_response(response, 'POST data without "locator" form data.')

    try:
        item = _get_item(request, request.POST)
    except (InvalidKeyError, ItemNotFoundError):
        return error_response(response, "Can't find item by locator.")

    if 'transcript-file' not in request.FILES:
        return error_response(response, 'POST data without "file" form data.')

    video_list = request.POST.get('video_list')
    if not video_list:
        return error_response(response, 'POST data without video names.')

    try:
        video_list = json.loads(video_list)
    except ValueError:
        return error_response(response, 'Invalid video_list JSON.')

    source_subs_filedata = request.FILES['transcript-file'].read().decode('utf8')
    source_subs_filename = request.FILES['transcript-file'].name

    if '.' not in source_subs_filename:
        return error_response(response, "Undefined file extension.")

    basename = os.path.basename(source_subs_filename)
    source_subs_name = os.path.splitext(basename)[0]
    source_subs_ext = os.path.splitext(basename)[1][1:]

    if item.category != 'video':
        return error_response(response, 'Transcripts are supported only for "video" modules.')

    # Allow upload only if any video link is presented
    if video_list:
        sub_attr = source_subs_name
        try:
            # Generate and save for 1.0 speed, will create subs_sub_attr.srt.sjson subtitles file in storage.
            generate_subs_from_source({1: sub_attr}, source_subs_ext, source_subs_filedata, item)

            for video_dict in video_list:
                video_name = video_dict['video']
                # We are creating transcripts for every video source, if in future some of video sources would be deleted.
                # Updates item.sub with `video_name` on success.
                copy_or_rename_transcript(video_name, sub_attr, item, user=request.user)

            response['subs'] = item.sub
            response['status'] = 'Success'
        except Exception as ex:
            return error_response(response, ex.message)
    else:
        return error_response(response, 'Empty video sources.')

    return JsonResponse(response)


@login_required
def download_transcripts(request):
    """
    Passes to user requested transcripts file.

    Raises Http404 if unsuccessful.
    """
    locator = request.GET.get('locator')
    if not locator:
        log.debug('GET data without "locator" property.')
        raise Http404

    try:
        item = _get_item(request, request.GET)
    except (InvalidKeyError, ItemNotFoundError):
        log.debug("Can't find item by locator.")
        raise Http404

    subs_id = request.GET.get('subs_id')
    if not subs_id:
        log.debug('GET data without "subs_id" property.')
        raise Http404

    if item.category != 'video':
        log.debug('transcripts are supported only for video" modules.')
        raise Http404

    filename = 'subs_{0}.srt.sjson'.format(subs_id)
    content_location = StaticContent.compute_location(item.location.course_key, filename)
    try:
        sjson_transcripts = contentstore().find(content_location)
        log.debug("Downloading subs for %s id", subs_id)
        str_subs = generate_srt_from_sjson(json.loads(sjson_transcripts.data), speed=1.0)
        if not str_subs:
            log.debug('generate_srt_from_sjson produces no subtitles')
            raise Http404
        response = HttpResponse(str_subs, content_type='application/x-subrip')
        response['Content-Disposition'] = 'attachment; filename="{0}.srt"'.format(subs_id)
        return response
    except NotFoundError:
        log.debug("Can't find content in storage for %s subs", subs_id)
        raise Http404


@login_required
def check_transcripts(request):
    """
    Check state of transcripts availability.

    request.GET['data'] has key `videos`, which can contain any of the following::

        [
            {u'type': u'youtube', u'video': u'OEoXaMPEzfM', u'mode': u'youtube'},
            {u'type': u'html5',    u'video': u'video1',             u'mode': u'mp4'}
            {u'type': u'html5',    u'video': u'video2',             u'mode': u'webm'}
        ]
        `type` is youtube or html5
        `video` is html5 or youtube video_id
        `mode` is youtube, ,p4 or webm

    Returns transcripts_presence dict::

        html5_local: list of html5 ids, if subtitles exist locally for them;
        is_youtube_mode: bool, if we have youtube_id, and as youtube mode is of higher priority, reflect this with flag;
        youtube_local: bool, if youtube transcripts exist locally;
        youtube_server: bool, if youtube transcripts exist on server;
        youtube_diff: bool, if youtube transcripts exist on youtube server, and are different from local youtube ones;
        current_item_subs: string, value of item.sub field;
        status: string, 'Error' or 'Success';
        subs: string, new value of item.sub field, that should be set in module;
        command: string, action to front-end what to do and what to show to user.
    """
    transcripts_presence = {
        'html5_local': [],
        'html5_equal': False,
        'is_youtube_mode': False,
        'youtube_local': False,
        'youtube_server': False,
        'youtube_diff': True,
        'current_item_subs': None,
        'status': 'Error',
    }
    try:
        __, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(transcripts_presence, e.message)

    transcripts_presence['status'] = 'Success'

    filename = 'subs_{0}.srt.sjson'.format(item.sub)
    content_location = StaticContent.compute_location(item.location.course_key, filename)
    try:
        local_transcripts = contentstore().find(content_location).data
        transcripts_presence['current_item_subs'] = item.sub
    except NotFoundError:
        pass

    # Check for youtube transcripts presence
    youtube_id = videos.get('youtube', None)
    if youtube_id:
        transcripts_presence['is_youtube_mode'] = True

        # youtube local
        filename = 'subs_{0}.srt.sjson'.format(youtube_id)
        content_location = StaticContent.compute_location(item.location.course_key, filename)
        try:
            local_transcripts = contentstore().find(content_location).data
            transcripts_presence['youtube_local'] = True
        except NotFoundError:
            log.debug("Can't find transcripts in storage for youtube id: %s", youtube_id)

        # youtube server
        youtube_text_api = copy.deepcopy(settings.YOUTUBE['TEXT_API'])
        youtube_text_api['params']['v'] = youtube_id
        youtube_response = requests.get('http://' + youtube_text_api['url'], params=youtube_text_api['params'])

        if youtube_response.status_code == 200 and youtube_response.text:
            transcripts_presence['youtube_server'] = True
        #check youtube local and server transcripts for equality
        if transcripts_presence['youtube_server'] and transcripts_presence['youtube_local']:
            try:
                youtube_server_subs = get_transcripts_from_youtube(
                    youtube_id,
                    settings,
                    item.runtime.service(item, "i18n")
                )
                if json.loads(local_transcripts) == youtube_server_subs:  # check transcripts for equality
                    transcripts_presence['youtube_diff'] = False
            except GetTranscriptsFromYouTubeException:
                pass

    # Check for html5 local transcripts presence
    html5_subs = []
    for html5_id in videos['html5']:
        filename = 'subs_{0}.srt.sjson'.format(html5_id)
        content_location = StaticContent.compute_location(item.location.course_key, filename)
        try:
            html5_subs.append(contentstore().find(content_location).data)
            transcripts_presence['html5_local'].append(html5_id)
        except NotFoundError:
            log.debug("Can't find transcripts in storage for non-youtube video_id: %s", html5_id)
        if len(html5_subs) == 2:  # check html5 transcripts for equality
            transcripts_presence['html5_equal'] = json.loads(html5_subs[0]) == json.loads(html5_subs[1])

    command, subs_to_use = _transcripts_logic(transcripts_presence, videos)
    transcripts_presence.update({
        'command': command,
        'subs': subs_to_use,
    })
    return JsonResponse(transcripts_presence)


def _transcripts_logic(transcripts_presence, videos):
    """
    By `transcripts_presence` content, figure what show to user:

    returns: `command` and `subs`.

    `command`: string,  action to front-end what to do and what show to user.
    `subs`: string, new value of item.sub field, that should be set in module.

    `command` is one of::

        replace: replace local youtube subtitles with server one's
        found: subtitles are found
        import: import subtitles from youtube server
        choose: choose one from two html5 subtitles
        not found: subtitles are not found
    """
    command = None

    # new value of item.sub field, that should be set in module.
    subs = ''

    # youtube transcripts are of high priority than html5 by design
    if (
            transcripts_presence['youtube_diff'] and
            transcripts_presence['youtube_local'] and
            transcripts_presence['youtube_server']):  # youtube server and local exist
        command = 'replace'
        subs = videos['youtube']
    elif transcripts_presence['youtube_local']:  # only youtube local exist
        command = 'found'
        subs = videos['youtube']
    elif transcripts_presence['youtube_server']:  # only youtube server exist
        command = 'import'
    else:  # html5 part
        if transcripts_presence['html5_local']:  # can be 1 or 2 html5 videos
            if len(transcripts_presence['html5_local']) == 1 or transcripts_presence['html5_equal']:
                command = 'found'
                subs = transcripts_presence['html5_local'][0]
            else:
                command = 'choose'
                subs = transcripts_presence['html5_local'][0]
        else:  # html5 source have no subtitles
            # check if item sub has subtitles
            if transcripts_presence['current_item_subs'] and not transcripts_presence['is_youtube_mode']:
                log.debug("Command is use existing %s subs", transcripts_presence['current_item_subs'])
                command = 'use_existing'
            else:
                command = 'not_found'
    log.debug(
        "Resulted command: %s, current transcripts: %s, youtube mode: %s",
        command,
        transcripts_presence['current_item_subs'],
        transcripts_presence['is_youtube_mode']
    )
    return command, subs


@login_required
def choose_transcripts(request):
    """
    Replaces html5 subtitles, presented for both html5 sources, with chosen one.

    Code removes rejected html5 subtitles and updates sub attribute with chosen html5_id.

    It does nothing with youtube id's.

    Returns: status `Success` and resulted item.sub value or status `Error` and HTTP 400.
    """
    response = {
        'status': 'Error',
        'subs': '',
    }

    try:
        data, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(response, e.message)

    html5_id = data.get('html5_id')  # html5_id chosen by user

    # find rejected html5_id and remove appropriate subs from store
    html5_id_to_remove = [x for x in videos['html5'] if x != html5_id]
    if html5_id_to_remove:
        remove_subs_from_store(html5_id_to_remove, item)

    if item.sub != html5_id:  # update sub value
        item.sub = html5_id
        item.save_with_metadata(request.user)
    response = {'status': 'Success',  'subs': item.sub}
    return JsonResponse(response)


@login_required
def replace_transcripts(request):
    """
    Replaces all transcripts with youtube ones.

    Downloads subtitles from youtube and replaces all transcripts with downloaded ones.

    Returns: status `Success` and resulted item.sub value or status `Error` and HTTP 400.
    """
    response = {'status': 'Error', 'subs': ''}

    try:
        __, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(response, e.message)

    youtube_id = videos['youtube']
    if not youtube_id:
        return error_response(response, 'YouTube id {} is not presented in request data.'.format(youtube_id))

    try:
        download_youtube_subs({1.0: youtube_id}, item, settings)
    except GetTranscriptsFromYouTubeException as e:
        return error_response(response, e.message)

    item.sub = youtube_id
    item.save_with_metadata(request.user)
    response = {'status': 'Success',  'subs': item.sub}
    return JsonResponse(response)


def _validate_transcripts_data(request):
    """
    Validates, that request contains all proper data for transcripts processing.

    Returns tuple of 3 elements::

        data: dict, loaded json from request,
        videos: parsed `data` to useful format,
        item:  video item from storage

    Raises `TranscriptsRequestValidationException` if validation is unsuccessful
    or `PermissionDenied` if user has no access.
    """
    data = json.loads(request.GET.get('data', '{}'))
    if not data:
        raise TranscriptsRequestValidationException(_('Incoming video data is empty.'))

    try:
        item = _get_item(request, data)
    except (InvalidKeyError, ItemNotFoundError):
        raise TranscriptsRequestValidationException(_("Can't find item by locator."))

    if item.category != 'video':
        raise TranscriptsRequestValidationException(_('Transcripts are supported only for "video" modules.'))

    # parse data form request.GET.['data']['video'] to useful format
    videos = {'youtube': '', 'html5': {}}
    for video_data in data.get('videos'):
        if video_data['type'] == 'youtube':
            videos['youtube'] = video_data['video']
        else:  # do not add same html5 videos
            if videos['html5'].get('video') != video_data['video']:
                videos['html5'][video_data['video']] = video_data['mode']

    return data, videos, item


@login_required
def rename_transcripts(request):
    """
    Create copies of existing subtitles with new names of HTML5 sources.

    Old subtitles are not deleted now, because we do not have rollback functionality.

    If succeed, Item.sub will be chosen randomly from html5 video sources provided by front-end.
    """
    response = {'status': 'Error', 'subs': ''}

    try:
        __, videos, item = _validate_transcripts_data(request)
    except TranscriptsRequestValidationException as e:
        return error_response(response, e.message)

    old_name = item.sub

    for new_name in videos['html5'].keys():  # copy subtitles for every HTML5 source
        try:
            # updates item.sub with new_name if it is successful.
            copy_or_rename_transcript(new_name, old_name, item, user=request.user)
        except NotFoundError:
            # subtitles file `item.sub` is not presented in the system. Nothing to copy or rename.
            error_response(response, "Can't find transcripts in storage for {}".format(old_name))

    response['status'] = 'Success'
    response['subs'] = item.sub  # item.sub has been changed, it is not equal to old_name.
    log.debug("Updated item.sub to %s", item.sub)
    return JsonResponse(response)


@login_required
def save_transcripts(request):
    """
    Saves video module with updated values of fields.

    Returns: status `Success` or status `Error` and HTTP 400.
    """
    response = {'status': 'Error'}

    data = json.loads(request.GET.get('data', '{}'))
    if not data:
        return error_response(response, 'Incoming video data is empty.')

    try:
        item = _get_item(request, data)
    except (InvalidKeyError, ItemNotFoundError):
        return error_response(response, "Can't find item by locator.")

    metadata = data.get('metadata')
    if metadata is not None:
        new_sub = metadata.get('sub')

        for metadata_key, value in metadata.items():
            setattr(item, metadata_key, value)

        item.save_with_metadata(request.user)  # item becomes updated with new values

        if new_sub:
            manage_video_subtitles_save(item, request.user)
        else:
            # If `new_sub` is empty, it means that user explicitly does not want to use
            # transcripts for current video ids and we remove all transcripts from storage.
            current_subs = data.get('current_subs')
            if current_subs is not None:
                for sub in current_subs:
                    remove_subs_from_store(sub, item)

        response['status'] = 'Success'

    return JsonResponse(response)


def _get_item(request, data):
    """
    Obtains from 'data' the locator for an item.
    Next, gets that item from the modulestore (allowing any errors to raise up).
    Finally, verifies that the user has access to the item.

    Returns the item.
    """
    usage_key = UsageKey.from_string(data.get('locator'))

    # This is placed before has_course_access() to validate the location,
    # because has_course_access() raises  r if location is invalid.
    item = modulestore().get_item(usage_key)

    if not has_course_access(request.user, usage_key.course_key):
        raise PermissionDenied()

    return item

########NEW FILE########
__FILENAME__ = user
from django.core.exceptions import PermissionDenied
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required
from django.views.decorators.http import require_http_methods
from django.utils.translation import ugettext as _
from django.views.decorators.http import require_POST
from django_future.csrf import ensure_csrf_cookie
from edxmako.shortcuts import render_to_response

from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from util.json_request import JsonResponse, expect_json
from student.roles import CourseInstructorRole, CourseStaffRole
from course_creators.views import user_requested_access

from .access import has_course_access

from student.models import CourseEnrollment
from django.http import HttpResponseNotFound
from student import auth


__all__ = ['request_course_creator', 'course_team_handler']


@require_POST
@login_required
def request_course_creator(request):
    """
    User has requested course creation access.
    """
    user_requested_access(request.user)
    return JsonResponse({"Status": "OK"})


# pylint: disable=unused-argument
@login_required
@ensure_csrf_cookie
@require_http_methods(("GET", "POST", "PUT", "DELETE"))
def course_team_handler(request, course_key_string=None, email=None):
    """
    The restful handler for course team users.

    GET
        html: return html page for managing course team
        json: return json representation of a particular course team member (email is required).
    POST or PUT
        json: modify the permissions for a particular course team member (email is required, as well as role in the payload).
    DELETE:
        json: remove a particular course team member from the course team (email is required).
    """
    course_key = CourseKey.from_string(course_key_string) if course_key_string else None
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
        return _course_team_user(request, course_key, email)
    elif request.method == 'GET':  # assume html
        return _manage_users(request, course_key)
    else:
        return HttpResponseNotFound()


def _manage_users(request, course_key):
    """
    This view will return all CMS users who are editors for the specified course
    """
    # check that logged in user has permissions to this item
    if not has_course_access(request.user, course_key):
        raise PermissionDenied()

    course_module = modulestore().get_course(course_key)
    instructors = CourseInstructorRole(course_key).users_with_role()
    # the page only lists staff and assumes they're a superset of instructors. Do a union to ensure.
    staff = set(CourseStaffRole(course_key).users_with_role()).union(instructors)

    return render_to_response('manage_users.html', {
        'context_course': course_module,
        'staff': staff,
        'instructors': instructors,
        'allow_actions': has_course_access(request.user, course_key, role=CourseInstructorRole),
    })


@expect_json
def _course_team_user(request, course_key, email):
    """
    Handle the add, remove, promote, demote requests ensuring the requester has authority
    """
    # check that logged in user has permissions to this item
    if has_course_access(request.user, course_key, role=CourseInstructorRole):
        # instructors have full permissions
        pass
    elif has_course_access(request.user, course_key, role=CourseStaffRole) and email == request.user.email:
        # staff can only affect themselves
        pass
    else:
        msg = {
            "error": _("Insufficient permissions")
        }
        return JsonResponse(msg, 400)


    try:
        user = User.objects.get(email=email)
    except Exception:
        msg = {
            "error": _("Could not find user by email address '{email}'.").format(email=email),
        }
        return JsonResponse(msg, 404)

    # role hierarchy: globalstaff > "instructor" > "staff" (in a course)
    if request.method == "GET":
        # just return info about the user
        msg = {
            "email": user.email,
            "active": user.is_active,
            "role": None,
        }
        # what's the highest role that this user has? (How should this report global staff?)
        for role in [CourseInstructorRole(course_key), CourseStaffRole(course_key)]:
            if role.has_user(user):
                msg["role"] = role.ROLE
                break
        return JsonResponse(msg)

    # can't modify an inactive user
    if not user.is_active:
        msg = {
            "error": _('User {email} has registered but has not yet activated his/her account.').format(email=email),
        }
        return JsonResponse(msg, 400)

    if request.method == "DELETE":
        try:
            try_remove_instructor(request, course_key, user)
        except CannotOrphanCourse as oops:
            return JsonResponse(oops.msg, 400)

        auth.remove_users(request.user, CourseStaffRole(course_key), user)
        return JsonResponse()

    # all other operations require the requesting user to specify a role
    role = request.json.get("role", request.POST.get("role"))
    if role is None:
        return JsonResponse({"error": _("`role` is required")}, 400)

    if role == "instructor":
        if not has_course_access(request.user, course_key, role=CourseInstructorRole):
            msg = {
                "error": _("Only instructors may create other instructors")
            }
            return JsonResponse(msg, 400)
        auth.add_users(request.user, CourseInstructorRole(course_key), user)
        # auto-enroll the course creator in the course so that "View Live" will work.
        CourseEnrollment.enroll(user, course_key)
    elif role == "staff":
        # add to staff regardless (can't do after removing from instructors as will no longer
        # be allowed)
        auth.add_users(request.user, CourseStaffRole(course_key), user)
        try:
            try_remove_instructor(request, course_key, user)
        except CannotOrphanCourse as oops:
            return JsonResponse(oops.msg, 400)

        # auto-enroll the course creator in the course so that "View Live" will work.
        CourseEnrollment.enroll(user, course_key)

    return JsonResponse()


class CannotOrphanCourse(Exception):
    """
    Exception raised if an attempt is made to remove all responsible instructors from course.
    """
    def __init__(self, msg):
        self.msg = msg
        Exception.__init__(self)


def try_remove_instructor(request, course_key, user):

    # remove all roles in this course from this user: but fail if the user
    # is the last instructor in the course team
    instructors = CourseInstructorRole(course_key)
    if instructors.has_user(user):
        if instructors.users_with_role().count() == 1:
            msg = {"error": _("You may not remove the last instructor from a course")}
            raise CannotOrphanCourse(msg)
        else:
            auth.remove_users(request.user, instructors, user)

########NEW FILE########
__FILENAME__ = xblock
"""
Views dedicated to rendering xblocks.
"""
from __future__ import absolute_import

import logging
import mimetypes

from xblock.core import XBlock

from django.conf import settings
from django.http import Http404, HttpResponse


log = logging.getLogger(__name__)


def xblock_resource(request, block_type, uri):  # pylint: disable=unused-argument
    """
    Return a package resource for the specified XBlock.
    """
    try:
        xblock_class = XBlock.load_class(block_type, settings.XBLOCK_SELECT_FUNCTION)
        content = xblock_class.open_local_resource(uri)
    except IOError:
        log.info('Failed to load xblock resource', exc_info=True)
        raise Http404
    except Exception:  # pylint: disable-msg=broad-except
        log.error('Failed to load xblock resource', exc_info=True)
        raise Http404

    mimetype, _ = mimetypes.guess_type(uri)
    return HttpResponse(content, mimetype=mimetype)

########NEW FILE########
__FILENAME__ = admin
"""
django admin page for the course creators table
"""

from course_creators.models import CourseCreator, update_creator_state, send_user_notification, send_admin_notification
from course_creators.views import update_course_creator_group

from ratelimitbackend import admin
from django.conf import settings
from django.dispatch import receiver
from edxmako.shortcuts import render_to_string
from django.core.mail import send_mail
from smtplib import SMTPException

import logging

log = logging.getLogger("studio.coursecreatoradmin")


def get_email(obj):
    """ Returns the email address for a user """
    return obj.user.email

get_email.short_description = 'email'


class CourseCreatorAdmin(admin.ModelAdmin):
    """
    Admin for the course creator table.
    """

    # Fields to display on the overview page.
    list_display = ['username', get_email, 'state', 'state_changed', 'note']
    readonly_fields = ['username', 'state_changed']
    # Controls the order on the edit form (without this, read-only fields appear at the end).
    fieldsets = (
        (None, {
            'fields': ['username', 'state', 'state_changed', 'note']
        }),
    )
    # Fields that filtering support
    list_filter = ['state', 'state_changed']
    # Fields that search supports.
    search_fields = ['user__username', 'user__email', 'state', 'note']
    # Turn off the action bar (we have no bulk actions)
    actions = None

    def username(self, inst):
        """
        Returns the username for a given user.

        Implemented to make sorting by username instead of by user object.
        """
        return inst.user.username

    username.admin_order_field = 'user__username'

    def has_add_permission(self, request):
        return False

    def has_delete_permission(self, request, obj=None):
        return False

    def has_change_permission(self, request, obj=None):
        return request.user.is_staff

    def save_model(self, request, obj, form, change):
        # Store who is making the request.
        obj.admin = request.user
        obj.save()


admin.site.register(CourseCreator, CourseCreatorAdmin)


@receiver(update_creator_state, sender=CourseCreator)
def update_creator_group_callback(sender, **kwargs):
    """
    Callback for when the model's creator status has changed.
    """
    user = kwargs['user']
    updated_state = kwargs['state']
    update_course_creator_group(kwargs['caller'], user, updated_state == CourseCreator.GRANTED)


@receiver(send_user_notification, sender=CourseCreator)
def send_user_notification_callback(sender, **kwargs):
    """
    Callback for notifying user about course creator status change.
    """
    user = kwargs['user']
    updated_state = kwargs['state']

    studio_request_email = settings.FEATURES.get('STUDIO_REQUEST_EMAIL', '')
    context = {'studio_request_email': studio_request_email}

    subject = render_to_string('emails/course_creator_subject.txt', context)
    subject = ''.join(subject.splitlines())
    if updated_state == CourseCreator.GRANTED:
        message_template = 'emails/course_creator_granted.txt'
    elif updated_state == CourseCreator.DENIED:
        message_template = 'emails/course_creator_denied.txt'
    else:
        # changed to unrequested or pending
        message_template = 'emails/course_creator_revoked.txt'
    message = render_to_string(message_template, context)

    try:
        user.email_user(subject, message, studio_request_email)
    except:
        log.warning("Unable to send course creator status e-mail to %s", user.email)


@receiver(send_admin_notification, sender=CourseCreator)
def send_admin_notification_callback(sender, **kwargs):
    """
    Callback for notifying admin of a user in the 'pending' state.
    """
    user = kwargs['user']

    studio_request_email = settings.FEATURES.get('STUDIO_REQUEST_EMAIL', '')
    context = {'user_name': user.username, 'user_email': user.email}

    subject = render_to_string('emails/course_creator_admin_subject.txt', context)
    subject = ''.join(subject.splitlines())
    message = render_to_string('emails/course_creator_admin_user_pending.txt', context)

    try:
        send_mail(
            subject,
            message,
            studio_request_email,
            [studio_request_email],
            fail_silently=False
        )
    except SMTPException:
        log.warning("Failure sending 'pending state' e-mail for %s to %s", user.email, studio_request_email)

########NEW FILE########
__FILENAME__ = 0001_initial
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'CourseCreator'
        db.create_table('course_creators_coursecreator', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], unique=True)),
            ('state_changed', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
            ('state', self.gf('django.db.models.fields.CharField')(default='unrequested', max_length=24)),
            ('note', self.gf('django.db.models.fields.CharField')(max_length=512, blank=True)),
        ))
        db.send_create_signal('course_creators', ['CourseCreator'])


    def backwards(self, orm):
        # Deleting model 'CourseCreator'
        db.delete_table('course_creators_coursecreator')


    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'course_creators.coursecreator': {
            'Meta': {'object_name': 'CourseCreator'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'note': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
            'state': ('django.db.models.fields.CharField', [], {'default': "'unrequested'", 'max_length': '24'}),
            'state_changed': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'unique': 'True'})
        }
    }

    complete_apps = ['course_creators']
########NEW FILE########
__FILENAME__ = models
"""
Table for storing information about whether or not Studio users have course creation privileges.
"""
from django.db import models
from django.db.models.signals import post_init, post_save
from django.dispatch import receiver, Signal
from django.contrib.auth.models import User

from django.utils import timezone
from django.utils.translation import ugettext as _

# A signal that will be sent when users should be added or removed from the creator group
update_creator_state = Signal(providing_args=["caller", "user", "state"])

# A signal that will be sent when admin should be notified of a pending user request
send_admin_notification = Signal(providing_args=["user"])

# A signal that will be sent when user should be notified of change in course creator privileges
send_user_notification = Signal(providing_args=["user", "state"])


class CourseCreator(models.Model):
    """
    Creates the database table model.
    """
    UNREQUESTED = 'unrequested'
    PENDING = 'pending'
    GRANTED = 'granted'
    DENIED = 'denied'

    # Second value is the "human-readable" version.
    STATES = (
        (UNREQUESTED, _(u'unrequested')),
        (PENDING, _(u'pending')),
        (GRANTED, _(u'granted')),
        (DENIED, _(u'denied')),
    )

    user = models.ForeignKey(User, help_text=_("Studio user"), unique=True)
    state_changed = models.DateTimeField('state last updated', auto_now_add=True,
                                         help_text=_("The date when state was last updated"))
    state = models.CharField(max_length=24, blank=False, choices=STATES, default=UNREQUESTED,
                             help_text=_("Current course creator state"))
    note = models.CharField(max_length=512, blank=True, help_text=_("Optional notes about this user (for example, "
                                                                    "why course creation access was denied)"))

    def __unicode__(self):
        return u"{0} | {1} [{2}]".format(self.user, self.state, self.state_changed)


@receiver(post_init, sender=CourseCreator)
def post_init_callback(sender, **kwargs):
    """
    Extend to store previous state.
    """
    instance = kwargs['instance']
    instance.orig_state = instance.state


@receiver(post_save, sender=CourseCreator)
def post_save_callback(sender, **kwargs):
    """
    Extend to update state_changed time and fire event to update course creator group, if appropriate.
    """
    instance = kwargs['instance']
    # We only wish to modify the state_changed time if the state has been modified. We don't wish to
    # modify it for changes to the notes field.
    if instance.state != instance.orig_state:
        granted_state_change = instance.state == CourseCreator.GRANTED or instance.orig_state == CourseCreator.GRANTED
        # If either old or new state is 'granted', we must manipulate the course creator
        # group maintained by authz. That requires staff permissions (stored admin).
        if granted_state_change:
            assert hasattr(instance, 'admin'), 'Must have stored staff user to change course creator group'
            update_creator_state.send(
                sender=sender,
                caller=instance.admin,
                user=instance.user,
                state=instance.state
            )

        # If user has been denied access, granted access, or previously granted access has been
        # revoked, send a notification message to the user.
        if instance.state == CourseCreator.DENIED or granted_state_change:
            send_user_notification.send(
                sender=sender,
                user=instance.user,
                state=instance.state
            )

        # If the user has gone into the 'pending' state, send a notification to interested admin.
        if instance.state == CourseCreator.PENDING:
            send_admin_notification.send(
                sender=sender,
                user=instance.user
            )

        instance.state_changed = timezone.now()
        instance.orig_state = instance.state
        instance.save()

########NEW FILE########
__FILENAME__ = test_admin
"""
Tests course_creators.admin.py.
"""

from django.test import TestCase
from django.contrib.auth.models import User
from django.contrib.admin.sites import AdminSite
from django.http import HttpRequest
import mock

from course_creators.admin import CourseCreatorAdmin
from course_creators.models import CourseCreator
from django.core import mail
from student.roles import CourseCreatorRole
from student import auth


def mock_render_to_string(template_name, context):
    """Return a string that encodes template_name and context"""
    return str((template_name, context))


class CourseCreatorAdminTest(TestCase):
    """
    Tests for course creator admin.
    """

    def setUp(self):
        """ Test case setup """
        self.user = User.objects.create_user('test_user', 'test_user+courses@edx.org', 'foo')
        self.table_entry = CourseCreator(user=self.user)
        self.table_entry.save()

        self.admin = User.objects.create_user('Mark', 'admin+courses@edx.org', 'foo')
        self.admin.is_staff = True

        self.request = HttpRequest()
        self.request.user = self.admin

        self.creator_admin = CourseCreatorAdmin(self.table_entry, AdminSite())

        self.studio_request_email = 'mark@marky.mark'
        self.enable_creator_group_patch = {
            "ENABLE_CREATOR_GROUP": True,
            "STUDIO_REQUEST_EMAIL": self.studio_request_email
        }

    @mock.patch('course_creators.admin.render_to_string', mock.Mock(side_effect=mock_render_to_string, autospec=True))
    @mock.patch('django.contrib.auth.models.User.email_user')
    def test_change_status(self, email_user):
        """
        Tests that updates to state impact the creator group maintained in authz.py and that e-mails are sent.
        """

        def change_state_and_verify_email(state, is_creator):
            """ Changes user state, verifies creator status, and verifies e-mail is sent based on transition """
            self._change_state(state)
            self.assertEqual(is_creator, auth.has_access(self.user, CourseCreatorRole()))

            context = {'studio_request_email': self.studio_request_email}
            if state == CourseCreator.GRANTED:
                template = 'emails/course_creator_granted.txt'
            elif state == CourseCreator.DENIED:
                template = 'emails/course_creator_denied.txt'
            else:
                template = 'emails/course_creator_revoked.txt'
            email_user.assert_called_with(
                mock_render_to_string('emails/course_creator_subject.txt', context),
                mock_render_to_string(template, context),
                self.studio_request_email
            )

        with mock.patch.dict('django.conf.settings.FEATURES', self.enable_creator_group_patch):

            # User is initially unrequested.
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))

            change_state_and_verify_email(CourseCreator.GRANTED, True)

            change_state_and_verify_email(CourseCreator.DENIED, False)

            change_state_and_verify_email(CourseCreator.GRANTED, True)

            change_state_and_verify_email(CourseCreator.PENDING, False)

            change_state_and_verify_email(CourseCreator.GRANTED, True)

            change_state_and_verify_email(CourseCreator.UNREQUESTED, False)

            change_state_and_verify_email(CourseCreator.DENIED, False)

    @mock.patch('course_creators.admin.render_to_string', mock.Mock(side_effect=mock_render_to_string, autospec=True))
    def test_mail_admin_on_pending(self):
        """
        Tests that the admin account is notified when a user is in the 'pending' state.
        """

        def check_admin_message_state(state, expect_sent_to_admin, expect_sent_to_user):
            """ Changes user state and verifies e-mail sent to admin address only when pending. """
            mail.outbox = []
            self._change_state(state)

            # If a message is sent to the user about course creator status change, it will be the first
            # message sent. Admin message will follow.
            base_num_emails = 1 if expect_sent_to_user else 0
            if expect_sent_to_admin:
                context = {'user_name': "test_user", 'user_email': 'test_user+courses@edx.org'}
                self.assertEquals(base_num_emails + 1, len(mail.outbox), 'Expected admin message to be sent')
                sent_mail = mail.outbox[base_num_emails]
                self.assertEquals(
                    mock_render_to_string('emails/course_creator_admin_subject.txt', context),
                    sent_mail.subject
                )
                self.assertEquals(
                    mock_render_to_string('emails/course_creator_admin_user_pending.txt', context),
                    sent_mail.body
                )
                self.assertEquals(self.studio_request_email, sent_mail.from_email)
                self.assertEqual([self.studio_request_email], sent_mail.to)
            else:
                self.assertEquals(base_num_emails, len(mail.outbox))

        with mock.patch.dict('django.conf.settings.FEATURES', self.enable_creator_group_patch):
            # E-mail message should be sent to admin only when new state is PENDING, regardless of what
            # previous state was (unless previous state was already PENDING).
            # E-mail message sent to user only on transition into and out of GRANTED state.
            check_admin_message_state(CourseCreator.UNREQUESTED, expect_sent_to_admin=False, expect_sent_to_user=False)
            check_admin_message_state(CourseCreator.PENDING, expect_sent_to_admin=True, expect_sent_to_user=False)
            check_admin_message_state(CourseCreator.GRANTED, expect_sent_to_admin=False, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.DENIED, expect_sent_to_admin=False, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.GRANTED, expect_sent_to_admin=False, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.PENDING, expect_sent_to_admin=True, expect_sent_to_user=True)
            check_admin_message_state(CourseCreator.PENDING, expect_sent_to_admin=False, expect_sent_to_user=False)
            check_admin_message_state(CourseCreator.DENIED, expect_sent_to_admin=False, expect_sent_to_user=True)

    def _change_state(self, state):
        """ Helper method for changing state """
        self.table_entry.state = state
        self.creator_admin.save_model(self.request, self.table_entry, None, True)

    def test_add_permission(self):
        """
        Tests that staff cannot add entries
        """
        self.assertFalse(self.creator_admin.has_add_permission(self.request))

    def test_delete_permission(self):
        """
        Tests that staff cannot delete entries
        """
        self.assertFalse(self.creator_admin.has_delete_permission(self.request))

    def test_change_permission(self):
        """
        Tests that only staff can change entries
        """
        self.assertTrue(self.creator_admin.has_change_permission(self.request))

        self.request.user = self.user
        self.assertFalse(self.creator_admin.has_change_permission(self.request))

    def test_rate_limit_login(self):
        with mock.patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}):
            post_params = {'username': self.user.username, 'password': 'wrong_password'}
            # try logging in 30 times, the default limit in the number of failed
            # login attempts in one 5 minute period before the rate gets limited
            for _ in xrange(30):
                response = self.client.post('/admin/', post_params)
                self.assertEquals(response.status_code, 200)

            response = self.client.post('/admin/', post_params)
            # Since we are using the default rate limit behavior, we are
            # expecting this to return a 403 error to indicate that there have
            # been too many attempts
            self.assertEquals(response.status_code, 403)

########NEW FILE########
__FILENAME__ = test_views
"""
Tests course_creators.views.py.
"""

from django.test import TestCase
from django.contrib.auth.models import User
from django.core.exceptions import PermissionDenied

from course_creators.views import add_user_with_status_unrequested, add_user_with_status_granted
from course_creators.views import get_course_creator_status, update_course_creator_group, user_requested_access
import mock
from student.roles import CourseCreatorRole
from student import auth


class CourseCreatorView(TestCase):
    """
    Tests for modifying the course creator table.
    """

    def setUp(self):
        """ Test case setup """
        self.user = User.objects.create_user('test_user', 'test_user+courses@edx.org', 'foo')
        self.admin = User.objects.create_user('Mark', 'admin+courses@edx.org', 'foo')
        self.admin.is_staff = True

    def test_staff_permission_required(self):
        """
        Tests that any method changing the course creator authz group must be called with staff permissions.
        """
        with self.assertRaises(PermissionDenied):
            add_user_with_status_granted(self.user, self.user)

        with self.assertRaises(PermissionDenied):
            update_course_creator_group(self.user, self.user, True)

    def test_table_initially_empty(self):
        self.assertIsNone(get_course_creator_status(self.user))

    def test_add_unrequested(self):
        add_user_with_status_unrequested(self.user)
        self.assertEqual('unrequested', get_course_creator_status(self.user))

        # Calling add again will be a no-op (even if state is different).
        add_user_with_status_granted(self.admin, self.user)
        self.assertEqual('unrequested', get_course_creator_status(self.user))

    def test_add_granted(self):
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            # Calling add_user_with_status_granted impacts is_user_in_course_group_role.
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))

            add_user_with_status_granted(self.admin, self.user)
            self.assertEqual('granted', get_course_creator_status(self.user))

            # Calling add again will be a no-op (even if state is different).
            add_user_with_status_unrequested(self.user)
            self.assertEqual('granted', get_course_creator_status(self.user))

            self.assertTrue(auth.has_access(self.user, CourseCreatorRole()))

    def test_update_creator_group(self):
        with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}):
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))
            update_course_creator_group(self.admin, self.user, True)
            self.assertTrue(auth.has_access(self.user, CourseCreatorRole()))
            update_course_creator_group(self.admin, self.user, False)
            self.assertFalse(auth.has_access(self.user, CourseCreatorRole()))

    def test_user_requested_access(self):
        add_user_with_status_unrequested(self.user)
        self.assertEqual('unrequested', get_course_creator_status(self.user))
        user_requested_access(self.user)
        self.assertEqual('pending', get_course_creator_status(self.user))

    def test_user_requested_already_granted(self):
        add_user_with_status_granted(self.admin, self.user)
        self.assertEqual('granted', get_course_creator_status(self.user))
        # Will not "downgrade" to pending because that would require removing the
        # user from the authz course creator group (and that can only be done by an admin).
        user_requested_access(self.user)
        self.assertEqual('granted', get_course_creator_status(self.user))

    def test_add_user_unrequested_staff(self):
        # Users marked as is_staff will not be added to the course creator table.
        add_user_with_status_unrequested(self.admin)
        self.assertIsNone(get_course_creator_status(self.admin))

    def test_add_user_granted_staff(self):
        # Users marked as is_staff will not be added to the course creator table.
        add_user_with_status_granted(self.admin, self.admin)
        self.assertIsNone(get_course_creator_status(self.admin))

########NEW FILE########
__FILENAME__ = views
"""
Methods for interacting programmatically with the user creator table.
"""
from course_creators.models import CourseCreator
from student.roles import CourseCreatorRole
from student import auth


def add_user_with_status_unrequested(user):
    """
    Adds a user to the course creator table with status 'unrequested'.

    If the user is already in the table, this method is a no-op
    (state will not be changed).

    If the user is marked as is_staff, this method is a no-op (user
    will not be added to table).
    """
    _add_user(user, CourseCreator.UNREQUESTED)


def add_user_with_status_granted(caller, user):
    """
    Adds a user to the course creator table with status 'granted'.

    If appropriate, this method also adds the user to the course creator group maintained by authz.py.
    Caller must have staff permissions.

    If the user is already in the table, this method is a no-op
    (state will not be changed).

    If the user is marked as is_staff, this method is a no-op (user
    will not be added to table, nor added to authz.py group).
    """
    if _add_user(user, CourseCreator.GRANTED):
        update_course_creator_group(caller, user, True)


def update_course_creator_group(caller, user, add):
    """
    Method for adding and removing users from the creator group.

    Caller must have staff permissions.
    """
    if add:
        auth.add_users(caller, CourseCreatorRole(), user)
    else:
        auth.remove_users(caller, CourseCreatorRole(), user)


def get_course_creator_status(user):
    """
    Returns the status for a particular user, or None if user is not in the table.

    Possible return values are:
        'unrequested' = user has not requested course creation rights
        'pending' = user has requested course creation rights
        'granted' = user has been granted course creation rights
        'denied' = user has been denied course creation rights
        None = user does not exist in the table
    """
    user = CourseCreator.objects.filter(user=user)
    if user.count() == 0:
        return None
    else:
        # User is defined to be unique, can assume a single entry.
        return user[0].state


def user_requested_access(user):
    """
    User has requested course creator access.

    This changes the user state to CourseCreator.PENDING, unless the user
    state is already CourseCreator.GRANTED, in which case this method is a no-op.
    """
    user = CourseCreator.objects.get(user=user)
    if user.state != CourseCreator.GRANTED:
        user.state = CourseCreator.PENDING
        user.save()


def _add_user(user, state):
    """
    Adds a user to the course creator table with the specified state.

    Returns True if user was added to table, else False.

    If the user is already in the table, this method is a no-op
    (state will not be changed, method will return False).

    If the user is marked as is_staff, this method is a no-op (False will be returned).
    """
    if not user.is_staff and CourseCreator.objects.filter(user=user).count() == 0:
        entry = CourseCreator(user=user, state=state)
        entry.save()
        return True

    return False

########NEW FILE########
__FILENAME__ = course_details
import re
import logging
import datetime
import json
from json.encoder import JSONEncoder

from xmodule.modulestore import Location
from xmodule.modulestore.exceptions import ItemNotFoundError
from contentstore.utils import get_modulestore, course_image_url
from models.settings import course_grading
from xmodule.fields import Date
from xmodule.modulestore.django import modulestore

class CourseDetails(object):
    def __init__(self, org, course_id, run):
        # still need these for now b/c the client's screen shows these 3 fields
        self.org = org
        self.course_id = course_id
        self.run = run
        self.start_date = None  # 'start'
        self.end_date = None  # 'end'
        self.enrollment_start = None
        self.enrollment_end = None
        self.syllabus = None  # a pdf file asset
        self.short_description = ""
        self.overview = ""  # html to render as the overview
        self.intro_video = None  # a video pointer
        self.effort = None  # int hours/week
        self.course_image_name = ""
        self.course_image_asset_path = ""  # URL of the course image

    @classmethod
    def fetch(cls, course_key):
        """
        Fetch the course details for the given course from persistence and return a CourseDetails model.
        """
        descriptor = modulestore('direct').get_course(course_key)
        course_details = cls(course_key.org, course_key.course, course_key.run)

        course_details.start_date = descriptor.start
        course_details.end_date = descriptor.end
        course_details.enrollment_start = descriptor.enrollment_start
        course_details.enrollment_end = descriptor.enrollment_end
        course_details.course_image_name = descriptor.course_image
        course_details.course_image_asset_path = course_image_url(descriptor)

        temploc = course_key.make_usage_key('about', 'syllabus')
        try:
            course_details.syllabus = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'short_description')
        try:
            course_details.short_description = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'overview')
        try:
            course_details.overview = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'effort')
        try:
            course_details.effort = get_modulestore(temploc).get_item(temploc).data
        except ItemNotFoundError:
            pass

        temploc = course_key.make_usage_key('about', 'video')
        try:
            raw_video = get_modulestore(temploc).get_item(temploc).data
            course_details.intro_video = CourseDetails.parse_video_tag(raw_video)
        except ItemNotFoundError:
            pass

        return course_details

    @classmethod
    def update_about_item(cls, course_key, about_key, data, course, user):
        """
        Update the about item with the new data blob. If data is None, then
        delete the about item.
        """
        temploc = course_key.make_usage_key('about', about_key)
        store = get_modulestore(temploc)
        if data is None:
            store.delete_item(temploc)
        else:
            try:
                about_item = store.get_item(temploc)
            except ItemNotFoundError:
                about_item = store.create_xmodule(temploc, system=course.runtime)
            about_item.data = data
            store.update_item(about_item, user.id)

    @classmethod
    def update_from_json(cls, course_key, jsondict, user):
        """
        Decode the json into CourseDetails and save any changed attrs to the db
        """
        module_store = modulestore('direct')
        descriptor = module_store.get_course(course_key)

        dirty = False

        # In the descriptor's setter, the date is converted to JSON using Date's to_json method.
        # Calling to_json on something that is already JSON doesn't work. Since reaching directly
        # into the model is nasty, convert the JSON Date to a Python date, which is what the
        # setter expects as input.
        date = Date()

        if 'start_date' in jsondict:
            converted = date.from_json(jsondict['start_date'])
        else:
            converted = None
        if converted != descriptor.start:
            dirty = True
            descriptor.start = converted

        if 'end_date' in jsondict:
            converted = date.from_json(jsondict['end_date'])
        else:
            converted = None

        if converted != descriptor.end:
            dirty = True
            descriptor.end = converted

        if 'enrollment_start' in jsondict:
            converted = date.from_json(jsondict['enrollment_start'])
        else:
            converted = None

        if converted != descriptor.enrollment_start:
            dirty = True
            descriptor.enrollment_start = converted

        if 'enrollment_end' in jsondict:
            converted = date.from_json(jsondict['enrollment_end'])
        else:
            converted = None

        if converted != descriptor.enrollment_end:
            dirty = True
            descriptor.enrollment_end = converted

        if 'course_image_name' in jsondict and jsondict['course_image_name'] != descriptor.course_image:
            descriptor.course_image = jsondict['course_image_name']
            dirty = True

        if dirty:
            module_store.update_item(descriptor, user.id)

        # NOTE: below auto writes to the db w/o verifying that any of the fields actually changed
        # to make faster, could compare against db or could have client send over a list of which fields changed.
        for about_type in ['syllabus', 'overview', 'effort', 'short_description']:
            cls.update_about_item(course_key, about_type, jsondict[about_type], descriptor, user)

        recomposed_video_tag = CourseDetails.recompose_video_tag(jsondict['intro_video'])
        cls.update_about_item(course_key, 'video', recomposed_video_tag, descriptor, user)

        # Could just return jsondict w/o doing any db reads, but I put the reads in as a means to confirm
        # it persisted correctly
        return CourseDetails.fetch(course_key)

    @staticmethod
    def parse_video_tag(raw_video):
        """
        Because the client really only wants the author to specify the youtube key, that's all we send to and get from the client.
        The problem is that the db stores the html markup as well (which, of course, makes any sitewide changes to how we do videos
        next to impossible.)
        """
        if not raw_video:
            return None

        keystring_matcher = re.search(r'(?<=embed/)[a-zA-Z0-9_-]+', raw_video)
        if keystring_matcher is None:
            keystring_matcher = re.search(r'<?=\d+:[a-zA-Z0-9_-]+', raw_video)

        if keystring_matcher:
            return keystring_matcher.group(0)
        else:
            logging.warn("ignoring the content because it doesn't not conform to expected pattern: " + raw_video)
            return None

    @staticmethod
    def recompose_video_tag(video_key):
        # TODO should this use a mako template? Of course, my hope is that this is a short-term workaround for the db not storing
        # the right thing
        result = None
        if video_key:
            result = '<iframe width="560" height="315" src="//www.youtube.com/embed/' + \
                video_key + '?rel=0" frameborder="0" allowfullscreen=""></iframe>'
        return result


# TODO move to a more general util?
class CourseSettingsEncoder(json.JSONEncoder):
    """
    Serialize CourseDetails, CourseGradingModel, datetime, and old Locations
    """
    def default(self, obj):
        if isinstance(obj, (CourseDetails, course_grading.CourseGradingModel)):
            return obj.__dict__
        elif isinstance(obj, Location):
            return obj.dict()
        elif isinstance(obj, datetime.datetime):
            return Date().to_json(obj)
        else:
            return JSONEncoder.default(self, obj)

########NEW FILE########
__FILENAME__ = course_grading
from datetime import timedelta
from xmodule.modulestore.django import modulestore
from xblock.fields import Scope


class CourseGradingModel(object):
    """
    Basically a DAO and Model combo for CRUD operations pertaining to grading policy.
    """
    # Within this class, allow access to protected members of client classes.
    # This comes up when accessing kvs data and caches during kvs saves and modulestore writes.
    def __init__(self, course_descriptor):
        self.graders = [
            CourseGradingModel.jsonize_grader(i, grader) for i, grader in enumerate(course_descriptor.raw_grader)
        ]  # weights transformed to ints [0..100]
        self.grade_cutoffs = course_descriptor.grade_cutoffs
        self.grace_period = CourseGradingModel.convert_set_grace_period(course_descriptor)

    @classmethod
    def fetch(cls, course_key):
        """
        Fetch the course grading policy for the given course from persistence and return a CourseGradingModel.
        """
        descriptor = modulestore('direct').get_course(course_key)
        model = cls(descriptor)
        return model

    @staticmethod
    def fetch_grader(course_key, index):
        """
        Fetch the course's nth grader
        Returns an empty dict if there's no such grader.
        """
        descriptor = modulestore('direct').get_course(course_key)
        index = int(index)
        if len(descriptor.raw_grader) > index:
            return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])

        # return empty model
        else:
            return {"id": index,
                    "type": "",
                    "min_count": 0,
                    "drop_count": 0,
                    "short_label": None,
                    "weight": 0
                    }

    @staticmethod
    def update_from_json(course_key, jsondict, user):
        """
        Decode the json into CourseGradingModel and save any changes. Returns the modified model.
        Probably not the usual path for updates as it's too coarse grained.
        """
        descriptor = modulestore('direct').get_course(course_key)

        graders_parsed = [CourseGradingModel.parse_grader(jsonele) for jsonele in jsondict['graders']]

        descriptor.raw_grader = graders_parsed
        descriptor.grade_cutoffs = jsondict['grade_cutoffs']

        modulestore('direct').update_item(descriptor, user.id)

        CourseGradingModel.update_grace_period_from_json(course_key, jsondict['grace_period'], user)

        return CourseGradingModel.fetch(course_key)

    @staticmethod
    def update_grader_from_json(course_key, grader, user):
        """
        Create or update the grader of the given type (string key) for the given course. Returns the modified
        grader which is a full model on the client but not on the server (just a dict)
        """
        descriptor = modulestore('direct').get_course(course_key)

        # parse removes the id; so, grab it before parse
        index = int(grader.get('id', len(descriptor.raw_grader)))
        grader = CourseGradingModel.parse_grader(grader)

        if index < len(descriptor.raw_grader):
            descriptor.raw_grader[index] = grader
        else:
            descriptor.raw_grader.append(grader)

        modulestore('direct').update_item(descriptor, user.id)

        return CourseGradingModel.jsonize_grader(index, descriptor.raw_grader[index])

    @staticmethod
    def update_cutoffs_from_json(course_key, cutoffs, user):
        """
        Create or update the grade cutoffs for the given course. Returns sent in cutoffs (ie., no extra
        db fetch).
        """
        descriptor = modulestore('direct').get_course(course_key)
        descriptor.grade_cutoffs = cutoffs

        modulestore('direct').update_item(descriptor, user.id)

        return cutoffs

    @staticmethod
    def update_grace_period_from_json(course_key, graceperiodjson, user):
        """
        Update the course's default grace period. Incoming dict is {hours: h, minutes: m} possibly as a
        grace_period entry in an enclosing dict. It is also safe to call this method with a value of
        None for graceperiodjson.
        """
        descriptor = modulestore('direct').get_course(course_key)

        # Before a graceperiod has ever been created, it will be None (once it has been
        # created, it cannot be set back to None).
        if graceperiodjson is not None:
            if 'grace_period' in graceperiodjson:
                graceperiodjson = graceperiodjson['grace_period']

            grace_timedelta = timedelta(**graceperiodjson)
            descriptor.graceperiod = grace_timedelta

            modulestore('direct').update_item(descriptor, user.id)

    @staticmethod
    def delete_grader(course_key, index, user):
        """
        Delete the grader of the given type from the given course.
        """
        descriptor = modulestore('direct').get_course(course_key)

        index = int(index)
        if index < len(descriptor.raw_grader):
            del descriptor.raw_grader[index]
            # force propagation to definition
            descriptor.raw_grader = descriptor.raw_grader

        modulestore('direct').update_item(descriptor, user.id)

    @staticmethod
    def delete_grace_period(course_key, user):
        """
        Delete the course's grace period.
        """
        descriptor = modulestore('direct').get_course(course_key)

        del descriptor.graceperiod

        modulestore('direct').update_item(descriptor, user.id)

    @staticmethod
    def get_section_grader_type(location):
        descriptor = modulestore('direct').get_item(location)
        return {
            "graderType": descriptor.format if descriptor.format is not None else 'notgraded',
            "location": unicode(location),
        }

    @staticmethod
    def update_section_grader_type(descriptor, grader_type, user):
        if grader_type is not None and grader_type != u'notgraded':
            descriptor.format = grader_type
            descriptor.graded = True
        else:
            del descriptor.format
            del descriptor.graded

        modulestore('direct').update_item(descriptor, user.id)
        return {'graderType': grader_type}

    @staticmethod
    def convert_set_grace_period(descriptor):
        # 5 hours 59 minutes 59 seconds => converted to iso format
        rawgrace = descriptor.graceperiod
        if rawgrace:
            hours_from_days = rawgrace.days * 24
            seconds = rawgrace.seconds
            hours_from_seconds = int(seconds / 3600)
            hours = hours_from_days + hours_from_seconds
            seconds -= hours_from_seconds * 3600
            minutes = int(seconds / 60)
            seconds -= minutes * 60

            graceperiod = {'hours': 0, 'minutes': 0, 'seconds': 0}
            if hours > 0:
                graceperiod['hours'] = hours

            if minutes > 0:
                graceperiod['minutes'] = minutes

            if seconds > 0:
                graceperiod['seconds'] = seconds

            return graceperiod
        else:
            return None

    @staticmethod
    def parse_grader(json_grader):
        # manual to clear out kruft
        result = {"type": json_grader["type"],
                  "min_count": int(json_grader.get('min_count', 0)),
                  "drop_count": int(json_grader.get('drop_count', 0)),
                  "short_label": json_grader.get('short_label', None),
                  "weight": float(json_grader.get('weight', 0)) / 100.0
                  }

        return result

    @staticmethod
    def jsonize_grader(i, grader):
        grader['id'] = i
        if grader['weight']:
            grader['weight'] *= 100
        if not 'short_label' in grader:
            grader['short_label'] = ""

        return grader

########NEW FILE########
__FILENAME__ = course_metadata
from xblock.fields import Scope

from contentstore.utils import get_modulestore
from cms.lib.xblock.mixin import CmsBlockMixin


class CourseMetadata(object):
    '''
    For CRUD operations on metadata fields which do not have specific editors
    on the other pages including any user generated ones.
    The objects have no predefined attrs but instead are obj encodings of the
    editable metadata.
    '''
    FILTERED_LIST = ['xml_attributes',
                     'start',
                     'end',
                     'enrollment_start',
                     'enrollment_end',
                     'tabs',
                     'graceperiod',
                     'checklists',
                     'show_timezone',
                     'format',
                     'graded',
    ]

    @classmethod
    def fetch(cls, descriptor):
        """
        Fetch the key:value editable course details for the given course from
        persistence and return a CourseMetadata model.
        """
        result = {}

        for field in descriptor.fields.values():
            if field.name in CmsBlockMixin.fields:
                continue

            if field.scope != Scope.settings:
                continue

            if field.name in cls.FILTERED_LIST:
                continue

            result[field.name] = field.read_json(descriptor)

        return result

    @classmethod
    def update_from_json(cls, descriptor, jsondict, filter_tabs=True, user=None):
        """
        Decode the json into CourseMetadata and save any changed attrs to the db.

        Ensures none of the fields are in the blacklist.
        """
        dirty = False

        # Copy the filtered list to avoid permanently changing the class attribute.
        filtered_list = list(cls.FILTERED_LIST)
        # Don't filter on the tab attribute if filter_tabs is False.
        if not filter_tabs:
            filtered_list.remove("tabs")

        for key, val in jsondict.iteritems():
            # should it be an error if one of the filtered list items is in the payload?
            if key in filtered_list:
                continue

            if key == "unsetKeys":
                dirty = True
                for unset in val:
                    descriptor.fields[unset].delete_from(descriptor)

            if hasattr(descriptor, key) and getattr(descriptor, key) != val:
                dirty = True
                value = descriptor.fields[key].from_json(val)
                setattr(descriptor, key, value)

        if dirty:
            get_modulestore(descriptor.location).update_item(descriptor, user.id if user else None)

        return cls.fetch(descriptor)

########NEW FILE########
__FILENAME__ = acceptance
"""
This config file extends the test environment configuration
so that we can run the lettuce acceptance tests.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .test import *
from lms.envs.sauce import *

# You need to start the server in debug mode,
# otherwise the browser will not render the pages correctly
DEBUG = True

# Output Django logs to a file
import logging
logging.basicConfig(filename=TEST_ROOT / "log" / "cms_acceptance.log", level=logging.ERROR)

import os


def seed():
    return os.getppid()

# Suppress error message "Cannot determine primary key of logged in user"
# from track.middleware that gets triggered when using an auto_auth workflow
# This is an ERROR level warning so we need to set the threshold at CRITICAL
logging.getLogger('track.middleware').setLevel(logging.CRITICAL)

# Suppress warning message "Cannot find corresponding link for name: <foo>"
# from edxmako.shortcuts. We have no appropriate pages in the platform to
# use, so these are not set up for TOS and PRIVACY
logging.getLogger('edxmako.shortcuts').setLevel(logging.ERROR)

DOC_STORE_CONFIG = {
    'host': 'localhost',
    'db': 'acceptance_xmodule',
    'collection': 'acceptance_modulestore_%s' % seed(),
}

MODULESTORE_OPTIONS = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'fs_root': TEST_ROOT / "data",
    'render_template': 'edxmako.shortcuts.render_to_string',
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'draft': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    }
}

CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'DOC_STORE_CONFIG': {
        'host': 'localhost',
        'db': 'acceptance_xcontent_%s' % seed(),
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

# Set this up so that rake lms[acceptance] and running the
# harvest command both use the same (test) database
# which they can flush without messing up your dev db
DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': TEST_ROOT / "db" / "test_edx.db",
        'TEST_NAME': TEST_ROOT / "db" / "test_edx.db"
    }
}

# Enable asset pipeline
# Our fork of django-pipeline uses `PIPELINE` instead of `PIPELINE_ENABLED`
# PipelineFinder is explained here: http://django-pipeline.readthedocs.org/en/1.1.24/storages.html
PIPELINE = True
STATICFILES_FINDERS += ('pipeline.finders.PipelineFinder', )

# Use the auto_auth workflow for creating users and logging them in
FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True

# Forums are disabled in test.py to speed up unit tests, but we do not have
# per-test control for lettuce acceptance tests.
# If you are writing an acceptance test that needs the discussion service enabled,
# do not write it in lettuce, but instead write it using bok-choy.
# DO NOT CHANGE THIS SETTING HERE.
FEATURES['ENABLE_DISCUSSION_SERVICE'] = False

# HACK
# Setting this flag to false causes imports to not load correctly in the lettuce python files
# We do not yet understand why this occurs. Setting this to true is a stopgap measure
USE_I18N = True

# Include the lettuce app for acceptance testing, including the 'harvest' django-admin command
INSTALLED_APPS += ('lettuce.django',)
LETTUCE_APPS = ('contentstore',)
LETTUCE_BROWSER = os.environ.get('LETTUCE_BROWSER', 'chrome')

# Where to run: local, saucelabs, or grid
LETTUCE_SELENIUM_CLIENT = os.environ.get('LETTUCE_SELENIUM_CLIENT', 'local')

SELENIUM_GRID = {
    'URL': 'http://127.0.0.1:4444/wd/hub',
    'BROWSER': LETTUCE_BROWSER,
}

#####################################################################
# Lastly, see if the developer has any local overrides.
try:
    from .private import *  # pylint: disable=F0401
except ImportError:
    pass

# Point the URL used to test YouTube availability to our stub YouTube server
YOUTUBE['API'] = "127.0.0.1:{0}/get_youtube_api/".format(YOUTUBE_PORT)
YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)

########NEW FILE########
__FILENAME__ = aws
"""
This is the default template for our main set of AWS servers.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

import json

from .common import *

from logsettings import get_logger_config
import os

from path import path
from dealer.git import git

# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
# configuration files are read during startup.
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)

# CONFIG_ROOT specifies the directory where the JSON configuration
# files are expected to be found. If not specified, use the project
# directory.
CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT))

# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
# based on the service variant. If no variant is use, don't use a
# prefix.
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""


############### ALWAYS THE SAME ################################

DEBUG = False
TEMPLATE_DEBUG = False

EMAIL_BACKEND = 'django_ses.SESBackend'
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage'

###################################### CELERY  ################################

# Don't use a connection pool, since connections are dropped by ELB.
BROKER_POOL_LIMIT = 0
BROKER_CONNECTION_TIMEOUT = 1

# For the Result Store, use the django cache named 'celery'
CELERY_RESULT_BACKEND = 'cache'
CELERY_CACHE_BACKEND = 'celery'

# When the broker is behind an ELB, use a heartbeat to refresh the
# connection and to detect if it has been dropped.
BROKER_HEARTBEAT = 10.0
BROKER_HEARTBEAT_CHECKRATE = 2

# Each worker should only fetch one message at a time
CELERYD_PREFETCH_MULTIPLIER = 1

# Skip djcelery migrations, since we don't use the database as the broker
SOUTH_MIGRATION_MODULES = {
    'djcelery': 'ignore',
}

# Rename the exchange and queues for each variant

QUEUE_VARIANT = CONFIG_PREFIX.lower()

CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT)

HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT)
DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT)
LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT)

CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE

CELERY_QUEUES = {
    HIGH_PRIORITY_QUEUE: {},
    LOW_PRIORITY_QUEUE: {},
    DEFAULT_PRIORITY_QUEUE: {}
}

############# NON-SECURE ENV CONFIG ##############################
# Things like server locations, ports, etc.
with open(CONFIG_ROOT / CONFIG_PREFIX + "env.json") as env_file:
    ENV_TOKENS = json.load(env_file)

# STATIC_URL_BASE specifies the base url to use for static files
STATIC_URL_BASE = ENV_TOKENS.get('STATIC_URL_BASE', None)
if STATIC_URL_BASE:
    # collectstatic will fail if STATIC_URL is a unicode string
    STATIC_URL = STATIC_URL_BASE.encode('ascii')
    if not STATIC_URL.endswith("/"):
        STATIC_URL += "/"
    STATIC_URL += git.revision + "/"

# GITHUB_REPO_ROOT is the base directory
# for course data
GITHUB_REPO_ROOT = ENV_TOKENS.get('GITHUB_REPO_ROOT', GITHUB_REPO_ROOT)

# STATIC_ROOT specifies the directory where static files are
# collected

STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None)
if STATIC_ROOT_BASE:
    STATIC_ROOT = path(STATIC_ROOT_BASE) / git.revision

EMAIL_BACKEND = ENV_TOKENS.get('EMAIL_BACKEND', EMAIL_BACKEND)
EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', None)

EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', EMAIL_HOST)
EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', EMAIL_PORT)
EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', EMAIL_USE_TLS)

LMS_BASE = ENV_TOKENS.get('LMS_BASE')
# Note that FEATURES['PREVIEW_LMS_BASE'] gets read in from the environment file.

SITE_NAME = ENV_TOKENS['SITE_NAME']

LOG_DIR = ENV_TOKENS['LOG_DIR']

CACHES = ENV_TOKENS['CACHES']
# Cache used for location mapping -- called many times with the same key/value
# in a given request.
if 'loc_cache' not in CACHES:
    CACHES['loc_cache'] = {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_location_mem_cache',
    }

SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
SESSION_ENGINE = ENV_TOKENS.get('SESSION_ENGINE', SESSION_ENGINE)

# allow for environments to specify what cookie name our login subsystem should use
# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can
# happen with some browsers (e.g. Firefox)
if ENV_TOKENS.get('SESSION_COOKIE_NAME', None):
    # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str()
    SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME'))

#Email overrides
DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL)
DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL)
ADMINS = ENV_TOKENS.get('ADMINS', ADMINS)
SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL)
MKTG_URLS = ENV_TOKENS.get('MKTG_URLS', MKTG_URLS)
TECH_SUPPORT_EMAIL = ENV_TOKENS.get('TECH_SUPPORT_EMAIL', TECH_SUPPORT_EMAIL)

COURSES_WITH_UNSAFE_CODE = ENV_TOKENS.get("COURSES_WITH_UNSAFE_CODE", [])

# Theme overrides
THEME_NAME = ENV_TOKENS.get('THEME_NAME', None)

#Timezone overrides
TIME_ZONE = ENV_TOKENS.get('TIME_ZONE', TIME_ZONE)

# Push to LMS overrides
GIT_REPO_EXPORT_DIR = ENV_TOKENS.get('GIT_REPO_EXPORT_DIR', '/edx/var/edxapp/export_course_repos')

# Translation overrides
LANGUAGES = ENV_TOKENS.get('LANGUAGES', LANGUAGES)
LANGUAGE_CODE = ENV_TOKENS.get('LANGUAGE_CODE', LANGUAGE_CODE)
USE_I18N = ENV_TOKENS.get('USE_I18N', USE_I18N)

ENV_FEATURES = ENV_TOKENS.get('FEATURES', ENV_TOKENS.get('MITX_FEATURES', {}))
for feature, value in ENV_FEATURES.items():
    FEATURES[feature] = value

WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED', WIKI_ENABLED)

LOGGING = get_logger_config(LOG_DIR,
                            logging_env=ENV_TOKENS['LOGGING_ENV'],
                            syslog_addr=(ENV_TOKENS['SYSLOG_SERVER'], 514),
                            debug=False,
                            service_variant=SERVICE_VARIANT)

#theming start:
PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME', 'edX')

# Event Tracking
if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
    TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")

# Django CAS external authentication settings
CAS_EXTRA_LOGIN_PARAMS = ENV_TOKENS.get("CAS_EXTRA_LOGIN_PARAMS", None)
if FEATURES.get('AUTH_USE_CAS'):
    CAS_SERVER_URL = ENV_TOKENS.get("CAS_SERVER_URL", None)
    AUTHENTICATION_BACKENDS = (
        'django.contrib.auth.backends.ModelBackend',
        'django_cas.backends.CASBackend',
    )
    INSTALLED_APPS += ('django_cas',)
    MIDDLEWARE_CLASSES += ('django_cas.middleware.CASMiddleware',)
    CAS_ATTRIBUTE_CALLBACK = ENV_TOKENS.get('CAS_ATTRIBUTE_CALLBACK', None)
    if CAS_ATTRIBUTE_CALLBACK:
        import importlib
        CAS_USER_DETAILS_RESOLVER = getattr(
            importlib.import_module(CAS_ATTRIBUTE_CALLBACK['module']),
            CAS_ATTRIBUTE_CALLBACK['function']
        )


################ SECURE AUTH ITEMS ###############################
# Secret things: passwords, access keys, etc.
with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.json") as auth_file:
    AUTH_TOKENS = json.load(auth_file)

EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', EMAIL_HOST_USER)
EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', EMAIL_HOST_PASSWORD)

# If Segment.io key specified, load it and turn on Segment.io if the feature flag is set
# Note that this is the Studio key. There is a separate key for the LMS.
SEGMENT_IO_KEY = AUTH_TOKENS.get('SEGMENT_IO_KEY')
if SEGMENT_IO_KEY:
    FEATURES['SEGMENT_IO'] = ENV_TOKENS.get('SEGMENT_IO', False)

AWS_ACCESS_KEY_ID = AUTH_TOKENS["AWS_ACCESS_KEY_ID"]
if AWS_ACCESS_KEY_ID == "":
    AWS_ACCESS_KEY_ID = None

AWS_SECRET_ACCESS_KEY = AUTH_TOKENS["AWS_SECRET_ACCESS_KEY"]
if AWS_SECRET_ACCESS_KEY == "":
    AWS_SECRET_ACCESS_KEY = None

DATABASES = AUTH_TOKENS['DATABASES']
MODULESTORE = AUTH_TOKENS['MODULESTORE']
CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
# Datadog for events!
DATADOG = AUTH_TOKENS.get("DATADOG", {})
DATADOG.update(ENV_TOKENS.get("DATADOG", {}))

# TODO: deprecated (compatibility with previous settings)
if 'DATADOG_API' in AUTH_TOKENS:
    DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API']

# Celery Broker
CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "")
CELERY_BROKER_HOSTNAME = ENV_TOKENS.get("CELERY_BROKER_HOSTNAME", "")
CELERY_BROKER_VHOST = ENV_TOKENS.get("CELERY_BROKER_VHOST", "")
CELERY_BROKER_USER = AUTH_TOKENS.get("CELERY_BROKER_USER", "")
CELERY_BROKER_PASSWORD = AUTH_TOKENS.get("CELERY_BROKER_PASSWORD", "")

BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT,
                                            CELERY_BROKER_USER,
                                            CELERY_BROKER_PASSWORD,
                                            CELERY_BROKER_HOSTNAME,
                                            CELERY_BROKER_VHOST)

# Event tracking
TRACKING_BACKENDS.update(AUTH_TOKENS.get("TRACKING_BACKENDS", {}))
EVENT_TRACKING_BACKENDS.update(AUTH_TOKENS.get("EVENT_TRACKING_BACKENDS", {}))

SUBDOMAIN_BRANDING = ENV_TOKENS.get('SUBDOMAIN_BRANDING', {})
VIRTUAL_UNIVERSITIES = ENV_TOKENS.get('VIRTUAL_UNIVERSITIES', [])

##### ACCOUNT LOCKOUT DEFAULT PARAMETERS #####
MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = ENV_TOKENS.get("MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED", 5)
MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = ENV_TOKENS.get("MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS", 15 * 60)

MICROSITE_CONFIGURATION = ENV_TOKENS.get('MICROSITE_CONFIGURATION', {})
MICROSITE_ROOT_DIR = path(ENV_TOKENS.get('MICROSITE_ROOT_DIR', ''))

#### PASSWORD POLICY SETTINGS #####
PASSWORD_MIN_LENGTH = ENV_TOKENS.get("PASSWORD_MIN_LENGTH")
PASSWORD_MAX_LENGTH = ENV_TOKENS.get("PASSWORD_MAX_LENGTH")
PASSWORD_COMPLEXITY = ENV_TOKENS.get("PASSWORD_COMPLEXITY", {})
PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = ENV_TOKENS.get("PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD")
PASSWORD_DICTIONARY = ENV_TOKENS.get("PASSWORD_DICTIONARY", [])

### INACTIVITY SETTINGS ####
SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = AUTH_TOKENS.get("SESSION_INACTIVITY_TIMEOUT_IN_SECONDS")

##### X-Frame-Options response header settings #####
X_FRAME_OPTIONS = ENV_TOKENS.get('X_FRAME_OPTIONS', X_FRAME_OPTIONS)

##### ADVANCED_SECURITY_CONFIG #####
ADVANCED_SECURITY_CONFIG = ENV_TOKENS.get('ADVANCED_SECURITY_CONFIG', {})

########NEW FILE########
__FILENAME__ = aws_migrate
"""
A Django settings file for use on AWS while running
database migrations, since we don't want to normally run the
LMS with enough privileges to modify the database schema.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

# Import everything from .aws so that our settings are based on those.
from .aws import *
import os
from django.core.exceptions import ImproperlyConfigured

DB_OVERRIDES = dict(
    PASSWORD=os.environ.get('DB_MIGRATION_PASS', None),
    ENGINE=os.environ.get('DB_MIGRATION_ENGINE', DATABASES['default']['ENGINE']),
    USER=os.environ.get('DB_MIGRATION_USER', DATABASES['default']['USER']),
    NAME=os.environ.get('DB_MIGRATION_NAME', DATABASES['default']['NAME']),
    HOST=os.environ.get('DB_MIGRATION_HOST', DATABASES['default']['HOST']),
    PORT=os.environ.get('DB_MIGRATION_PORT', DATABASES['default']['PORT']),
)

if DB_OVERRIDES['PASSWORD'] is None:
    raise ImproperlyConfigured("No database password was provided for running "
                               "migrations.  This is fatal.")

for override, value in DB_OVERRIDES.iteritems():
    DATABASES['default'][override] = value

########NEW FILE########
__FILENAME__ = bok_choy
"""
Settings for bok choy tests
"""

import os
from path import path


########################## Prod-like settings ###################################
# These should be as close as possible to the settings we use in production.
# As in prod, we read in environment and auth variables from JSON files.
# Unlike in prod, we use the JSON files stored in this repo.
# This is a convenience for ensuring (a) that we can consistently find the files
# and (b) that the files are the same in Jenkins as in local dev.
os.environ['SERVICE_VARIANT'] = 'bok_choy'
os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname()  # pylint: disable=E1120

from .aws import *  # pylint: disable=W0401, W0614

######################### Testing overrides ####################################

# Needed for the reset database management command
INSTALLED_APPS += ('django_extensions',)

# Redirect to the test_root folder within the repo
TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root"  # pylint: disable=E1120
GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
LOG_DIR = (TEST_ROOT / "log").abspath()

# Configure Mongo modulestore to use the test folder within the repo
for store in ["default", "direct"]:
    MODULESTORE[store]['OPTIONS']['fs_root'] = (TEST_ROOT / "data").abspath()  # pylint: disable=E1120

# Enable django-pipeline and staticfiles
STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
PIPELINE = True

# Silence noisy logs
import logging
LOG_OVERRIDES = [
    ('track.middleware', logging.CRITICAL),
    ('edx.discussion', logging.CRITICAL),
]
for log_name, log_level in LOG_OVERRIDES:
    logging.getLogger(log_name).setLevel(log_level)

# Use the auto_auth workflow for creating users and logging them in
FEATURES['AUTOMATIC_AUTH_FOR_TESTING'] = True

# Unfortunately, we need to use debug mode to serve staticfiles
DEBUG = True

# Point the URL used to test YouTube availability to our stub YouTube server
YOUTUBE_PORT = 9080
YOUTUBE['API'] = "127.0.0.1:{0}/get_youtube_api/".format(YOUTUBE_PORT)
YOUTUBE['TEST_URL'] = "127.0.0.1:{0}/test_youtube/".format(YOUTUBE_PORT)
YOUTUBE['TEXT_API']['url'] = "127.0.0.1:{0}/test_transcripts_youtube/".format(YOUTUBE_PORT)

########NEW FILE########
__FILENAME__ = common
# -*- coding: utf-8 -*-
"""
This is the common settings file, intended to set sane defaults. If you have a
piece of configuration that's dependent on a set of feature flags being set,
then create a function that returns the calculated value based on the value of
FEATURES[...]. Modules that extend this one can change the feature
configuration in an environment specific config file and re-calculate those
values.

We should make a method that calls all these config methods so that you just
make one call at the end of your site-specific dev file to reset all the
dependent variables (like INSTALLED_APPS) for you.

Longer TODO:
1. Right now our treatment of static content in general and in particular
   course-specific static content is haphazard.
2. We should have a more disciplined approach to feature flagging, even if it
   just means that we stick them in a dict called FEATURES.
3. We need to handle configuration for multiple courses. This could be as
   multiple sites, but we do need a way to map their data assets.
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0611, W0614

import imp
import sys
import lms.envs.common
from lms.envs.common import (
    USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED
)
from path import path

from lms.lib.xblock.mixin import LmsBlockMixin
from cms.lib.xblock.mixin import CmsBlockMixin
from dealer.git import git

############################ FEATURE CONFIGURATION #############################

FEATURES = {
    'USE_DJANGO_PIPELINE': True,

    'GITHUB_PUSH': False,

    # for consistency in user-experience, keep the value of the following 3 settings
    # in sync with the ones in lms/envs/common.py
    'ENABLE_DISCUSSION_SERVICE': True,
    'ENABLE_TEXTBOOK': True,
    'ENABLE_STUDENT_NOTES': True,

    'AUTH_USE_CERTIFICATES': False,

    # email address for studio staff (eg to request course creation)
    'STUDIO_REQUEST_EMAIL': '',

    'STUDIO_NPS_SURVEY': True,

    # Segment.io - must explicitly turn it on for production
    'SEGMENT_IO': False,

    # Enable URL that shows information about the status of various services
    'ENABLE_SERVICE_STATUS': False,

    # Don't autoplay videos for course authors
    'AUTOPLAY_VIDEOS': False,

    # If set to True, new Studio users won't be able to author courses unless
    # edX has explicitly added them to the course creator group.
    'ENABLE_CREATOR_GROUP': False,

    # whether to use password policy enforcement or not
    'ENFORCE_PASSWORD_POLICY': False,

    # If set to True, Studio won't restrict the set of advanced components
    # to just those pre-approved by edX
    'ALLOW_ALL_ADVANCED_COMPONENTS': False,

    # Turn off account locking if failed login attempts exceeds a limit
    'ENABLE_MAX_FAILED_LOGIN_ATTEMPTS': False,

    # Allow editing of short description in course settings in cms
    'EDITABLE_SHORT_DESCRIPTION': True,

    # Hide any Personally Identifiable Information from application logs
    'SQUELCH_PII_IN_LOGS': False,

    # Toggles the embargo functionality, which enable embargoing for particular courses
    'EMBARGO': False,

    # Toggles the embargo site functionality, which enable embargoing for the whole site
    'SITE_EMBARGOED': False,

    # Turn on/off Microsites feature
    'USE_MICROSITES': False,

    # Allow creating courses with non-ascii characters in the course id
    'ALLOW_UNICODE_COURSE_ID': False,

    # Prevent concurrent logins per user
    'PREVENT_CONCURRENT_LOGINS': False,

    # Turn off Advanced Security by default
    'ADVANCED_SECURITY': False,
}
ENABLE_JASMINE = False


############################# SET PATH INFORMATION #############################
PROJECT_ROOT = path(__file__).abspath().dirname().dirname()  # /edx-platform/cms
REPO_ROOT = PROJECT_ROOT.dirname()
COMMON_ROOT = REPO_ROOT / "common"
LMS_ROOT = REPO_ROOT / "lms"
ENV_ROOT = REPO_ROOT.dirname()  # virtualenv dir /edx-platform is in

GITHUB_REPO_ROOT = ENV_ROOT / "data"

sys.path.append(REPO_ROOT)
sys.path.append(PROJECT_ROOT / 'djangoapps')
sys.path.append(COMMON_ROOT / 'djangoapps')
sys.path.append(COMMON_ROOT / 'lib')

# For geolocation ip database
GEOIP_PATH = REPO_ROOT / "common/static/data/geoip/GeoIP.dat"


############################# WEB CONFIGURATION #############################
# This is where we stick our compiled template files.
from tempdir import mkdtemp_clean
MAKO_MODULE_DIR = mkdtemp_clean('mako')
MAKO_TEMPLATES = {}
MAKO_TEMPLATES['main'] = [
    PROJECT_ROOT / 'templates',
    COMMON_ROOT / 'templates',
    COMMON_ROOT / 'djangoapps' / 'pipeline_mako' / 'templates',
    COMMON_ROOT / 'djangoapps' / 'pipeline_js' / 'templates',
]

for namespace, template_dirs in lms.envs.common.MAKO_TEMPLATES.iteritems():
    MAKO_TEMPLATES['lms.' + namespace] = template_dirs

TEMPLATE_DIRS = MAKO_TEMPLATES['main']

EDX_ROOT_URL = ''

LOGIN_REDIRECT_URL = EDX_ROOT_URL + '/signin'
LOGIN_URL = EDX_ROOT_URL + '/signin'


TEMPLATE_CONTEXT_PROCESSORS = (
    'django.core.context_processors.request',
    'django.core.context_processors.static',
    'django.contrib.messages.context_processors.messages',
    'django.core.context_processors.i18n',
    'django.contrib.auth.context_processors.auth',  # this is required for admin
    'django.core.context_processors.csrf',
    'dealer.contrib.django.staff.context_processor',  # access git revision
    'contentstore.context_processors.doc_url',
)

# use the ratelimit backend to prevent brute force attacks
AUTHENTICATION_BACKENDS = (
    'ratelimitbackend.backends.RateLimitModelBackend',
)

LMS_BASE = None

#################### CAPA External Code Evaluation #############################
XQUEUE_INTERFACE = {
    'url': 'http://localhost:8888',
    'django_auth': {'username': 'local',
                    'password': 'local'},
    'basic_auth': None,
}


################################# Middleware ###################################
# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'staticfiles.finders.FileSystemFinder',
    'staticfiles.finders.AppDirectoriesFinder',
    'pipeline.finders.PipelineFinder',
)

# List of callables that know how to import templates from various sources.
TEMPLATE_LOADERS = (
    'django.template.loaders.filesystem.Loader',
    'django.template.loaders.app_directories.Loader',
)

MIDDLEWARE_CLASSES = (
    'request_cache.middleware.RequestCache',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'method_override.middleware.MethodOverrideMiddleware',

    # Instead of AuthenticationMiddleware, we use a cache-backed version
    'cache_toolbox.middleware.CacheBackedAuthenticationMiddleware',
    'student.middleware.UserStandingMiddleware',
    'contentserver.middleware.StaticContentServer',
    'crum.CurrentRequestUserMiddleware',

    'django.contrib.messages.middleware.MessageMiddleware',
    'track.middleware.TrackMiddleware',

    # Allows us to dark-launch particular languages
    'dark_lang.middleware.DarkLangMiddleware',

    'embargo.middleware.EmbargoMiddleware',

    # Detects user-requested locale from 'accept-language' header in http request
    'django.middleware.locale.LocaleMiddleware',

    'django.middleware.transaction.TransactionMiddleware',
    # needs to run after locale middleware (or anything that modifies the request context)
    'edxmako.middleware.MakoMiddleware',

    # catches any uncaught RateLimitExceptions and returns a 403 instead of a 500
    'ratelimitbackend.middleware.RateLimitMiddleware',

    # for expiring inactive sessions
    'session_inactivity_timeout.middleware.SessionInactivityTimeout',

    # use Django built in clickjacking protection
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

# Clickjacking protection can be enabled by setting this to 'DENY'
X_FRAME_OPTIONS = 'ALLOW'

############# XBlock Configuration ##########

# Import after sys.path fixup
from xmodule.modulestore.inheritance import InheritanceMixin
from xmodule.modulestore import prefer_xmodules
from xmodule.x_module import XModuleMixin

# This should be moved into an XBlock Runtime/Application object
# once the responsibility of XBlock creation is moved out of modulestore - cpennington
XBLOCK_MIXINS = (LmsBlockMixin, CmsBlockMixin, InheritanceMixin, XModuleMixin)

# Allow any XBlock in Studio
# You should also enable the ALLOW_ALL_ADVANCED_COMPONENTS feature flag, so that
# xblocks can be added via advanced settings
XBLOCK_SELECT_FUNCTION = prefer_xmodules

############################ DJANGO_BUILTINS ################################
# Change DEBUG/TEMPLATE_DEBUG in your environment settings files, not here
DEBUG = False
TEMPLATE_DEBUG = False

# Site info
SITE_ID = 1
SITE_NAME = "localhost:8001"
HTTPS = 'on'
ROOT_URLCONF = 'cms.urls'

# Email
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
EMAIL_HOST = 'localhost'
EMAIL_PORT = 25
EMAIL_USE_TLS = False
EMAIL_HOST_USER = ''
EMAIL_HOST_PASSWORD = ''
DEFAULT_FROM_EMAIL = 'registration@example.com'
DEFAULT_FEEDBACK_EMAIL = 'feedback@example.com'
SERVER_EMAIL = 'devops@example.com'
ADMINS = ()
MANAGERS = ADMINS

# Static content
STATIC_URL = '/static/' + git.revision + "/"
ADMIN_MEDIA_PREFIX = '/static/admin/'
STATIC_ROOT = ENV_ROOT / "staticfiles" / git.revision

STATICFILES_DIRS = [
    COMMON_ROOT / "static",
    PROJECT_ROOT / "static",
    LMS_ROOT / "static",

    # This is how you would use the textbook images locally
    # ("book", ENV_ROOT / "book_images"),
]

# Locale/Internationalization
TIME_ZONE = 'America/New_York'  # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
LANGUAGE_CODE = 'en'  # http://www.i18nguy.com/unicode/language-identifiers.html

LANGUAGES = lms.envs.common.LANGUAGES
USE_I18N = True
USE_L10N = True

# Localization strings (e.g. django.po) are under this directory
LOCALE_PATHS = (REPO_ROOT + '/conf/locale',)  # edx-platform/conf/locale/

# Messages
MESSAGE_STORAGE = 'django.contrib.messages.storage.session.SessionStorage'

##### EMBARGO #####
EMBARGO_SITE_REDIRECT_URL = None

############################### Pipeline #######################################

STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'

from rooted_paths import rooted_glob

PIPELINE_CSS = {
    'style-vendor': {
        'source_filenames': [
            'css/vendor/normalize.css',
            'css/vendor/font-awesome.css',
            'css/vendor/html5-input-polyfills/number-polyfill.css',
            'js/vendor/CodeMirror/codemirror.css',
            'css/vendor/ui-lightness/jquery-ui-1.8.22.custom.css',
            'css/vendor/jquery.qtip.min.css',
            'js/vendor/markitup/skins/simple/style.css',
            'js/vendor/markitup/sets/wiki/style.css',
        ],
        'output_filename': 'css/cms-style-vendor.css',
    },
    'style-vendor-tinymce-content': {
        'source_filenames': [
            'css/tinymce-studio-content-fonts.css',
            'js/vendor/tinymce/js/tinymce/skins/studio-tmce4/content.min.css',
            'css/tinymce-studio-content.css'
        ],
        'output_filename': 'css/cms-style-vendor-tinymce-content.css',
    },
    'style-vendor-tinymce-skin': {
        'source_filenames': [
            'js/vendor/tinymce/js/tinymce/skins/studio-tmce4/skin.min.css'
        ],
        'output_filename': 'css/cms-style-vendor-tinymce-skin.css',
    },
    'style-app': {
        'source_filenames': [
            'sass/style-app.css',
        ],
        'output_filename': 'css/cms-style-app.css',
    },
    'style-app-extend1': {
        'source_filenames': [
            'sass/style-app-extend1.css',
        ],
        'output_filename': 'css/cms-style-app-extend1.css',
    },
    'style-xmodule': {
        'source_filenames': [
            'sass/style-xmodule.css',
        ],
        'output_filename': 'css/cms-style-xmodule.css',
    },
}

# test_order: Determines the position of this chunk of javascript on
# the jasmine test page
PIPELINE_JS = {
    'module-js': {
        'source_filenames': (
            rooted_glob(COMMON_ROOT / 'static/', 'xmodule/descriptors/js/*.js') +
            rooted_glob(COMMON_ROOT / 'static/', 'xmodule/modules/js/*.js') +
            rooted_glob(COMMON_ROOT / 'static/', 'coffee/src/discussion/*.js')
        ),
        'output_filename': 'js/cms-modules.js',
        'test_order': 1
    },
}

PIPELINE_COMPILERS = (
    'pipeline.compilers.coffee.CoffeeScriptCompiler',
)

PIPELINE_CSS_COMPRESSOR = None
PIPELINE_JS_COMPRESSOR = None

STATICFILES_IGNORE_PATTERNS = (
    "*.py",
    "*.pyc"
    # it would be nice if we could do, for example, "**/*.scss",
    # but these strings get passed down to the `fnmatch` module,
    # which doesn't support that. :(
    # http://docs.python.org/2/library/fnmatch.html
    "sass/*.scss",
    "sass/*/*.scss",
    "sass/*/*/*.scss",
    "sass/*/*/*/*.scss",
    "coffee/*.coffee",
    "coffee/*/*.coffee",
    "coffee/*/*/*.coffee",
    "coffee/*/*/*/*.coffee",

    # Symlinks used by js-test-tool
    "xmodule_js",
    "common_static",
)

PIPELINE_YUI_BINARY = 'yui-compressor'

################################# CELERY ######################################

# Message configuration

CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'

CELERY_MESSAGE_COMPRESSION = 'gzip'

# Results configuration

CELERY_IGNORE_RESULT = False
CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True

# Events configuration

CELERY_TRACK_STARTED = True

CELERY_SEND_EVENTS = True
CELERY_SEND_TASK_SENT_EVENT = True

# Exchange configuration

CELERY_DEFAULT_EXCHANGE = 'edx.core'
CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'

# Queues configuration

HIGH_PRIORITY_QUEUE = 'edx.core.high'
DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
LOW_PRIORITY_QUEUE = 'edx.core.low'

CELERY_QUEUE_HA_POLICY = 'all'

CELERY_CREATE_MISSING_QUEUES = True

CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE

CELERY_QUEUES = {
    HIGH_PRIORITY_QUEUE: {},
    LOW_PRIORITY_QUEUE: {},
    DEFAULT_PRIORITY_QUEUE: {}
}


############################## Video ##########################################

YOUTUBE = {
    # YouTube JavaScript API
    'API': 'www.youtube.com/iframe_api',

    # URL to test YouTube availability
    'TEST_URL': 'gdata.youtube.com/feeds/api/videos/',

    # Current youtube api for requesting transcripts.
    # For example: http://video.google.com/timedtext?lang=en&v=j_jEn79vS3g.
    'TEXT_API': {
        'url': 'video.google.com/timedtext',
        'params': {
            'lang': 'en',
            'v': 'set_youtube_id_of_11_symbols_here',
        },
    },
}

############################ APPS #####################################

INSTALLED_APPS = (
    # Standard apps
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'djcelery',
    'south',
    'method_override',

    # Database-backed configuration
    'config_models',

    # Monitor the status of services
    'service_status',

    # Testing
    'django_nose',

    # For CMS
    'contentstore',
    'course_creators',
    'student',  # misleading name due to sharing with lms
    'course_groups',  # not used in cms (yet), but tests run

    # Tracking
    'track',
    'eventtracking.django',

    # Monitoring
    'datadog',

    # For asset pipelining
    'edxmako',
    'pipeline',
    'staticfiles',
    'static_replace',

    # comment common
    'django_comment_common',

    # for course creator table
    'django.contrib.admin',

    # for managing course modes
    'course_modes',

    # Dark-launching languages
    'dark_lang',

    # Student identity reverification
    'reverification',

    # User preferences
    'user_api',
    'django_openid_auth',

    'embargo',

    # Monitoring signals
    'monitoring',
)


################# EDX MARKETING SITE ##################################

EDXMKTG_COOKIE_NAME = 'edxloggedin'
MKTG_URLS = {}
MKTG_URL_LINK_MAP = {

}

COURSES_WITH_UNSAFE_CODE = []

############################## EVENT TRACKING #################################

TRACK_MAX_EVENT = 50000

TRACKING_BACKENDS = {
    'logger': {
        'ENGINE': 'track.backends.logger.LoggerBackend',
        'OPTIONS': {
            'name': 'tracking'
        }
    }
}

# We're already logging events, and we don't want to capture user
# names/passwords.  Heartbeat events are likely not interesting.
TRACKING_IGNORE_URL_PATTERNS = [r'^/event', r'^/login', r'^/heartbeat']

EVENT_TRACKING_ENABLED = True
EVENT_TRACKING_BACKENDS = {
    'logger': {
        'ENGINE': 'eventtracking.backends.logger.LoggerBackend',
        'OPTIONS': {
            'name': 'tracking',
            'max_event_size': TRACK_MAX_EVENT,
        }
    }
}
EVENT_TRACKING_PROCESSORS = [
    {
        'ENGINE': 'track.shim.LegacyFieldMappingProcessor'
    }
]

#### PASSWORD POLICY SETTINGS #####

PASSWORD_MIN_LENGTH = None
PASSWORD_MAX_LENGTH = None
PASSWORD_COMPLEXITY = {}
PASSWORD_DICTIONARY_EDIT_DISTANCE_THRESHOLD = None
PASSWORD_DICTIONARY = []

##### ACCOUNT LOCKOUT DEFAULT PARAMETERS #####
MAX_FAILED_LOGIN_ATTEMPTS_ALLOWED = 5
MAX_FAILED_LOGIN_ATTEMPTS_LOCKOUT_PERIOD_SECS = 15 * 60


### Apps only installed in some instances

OPTIONAL_APPS = (
    'edx_jsdraw',
    'mentoring',

    # edx-ora2
    'submissions',
    'openassessment',
    'openassessment.assessment',
    'openassessment.workflow',
    'openassessment.xblock'
)


for app_name in OPTIONAL_APPS:
    # First attempt to only find the module rather than actually importing it,
    # to avoid circular references - only try to import if it can't be found
    # by find_module, which doesn't work with import hooks
    try:
        imp.find_module(app_name)
    except ImportError:
        try:
            __import__(app_name)
        except ImportError:
            continue
    INSTALLED_APPS += (app_name,)

### ADVANCED_SECURITY_CONFIG
# Empty by default
ADVANCED_SECURITY_CONFIG = {}

########NEW FILE########
__FILENAME__ = debug_upload
# pylint: disable=W0614, W0401
from .dev import *

FILE_UPLOAD_HANDLERS = (
    'contentstore.debug_file_uploader.DebugFileUploader',
    'django.core.files.uploadhandler.MemoryFileUploadHandler',
    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
)

########NEW FILE########
__FILENAME__ = dev
"""
This config file runs the simplest dev environment"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .common import *
from logsettings import get_logger_config

# import settings from LMS for consistent behavior with CMS
from lms.envs.dev import (WIKI_ENABLED)

DEBUG = True
TEMPLATE_DEBUG = DEBUG
LOGGING = get_logger_config(ENV_ROOT / "log",
                            logging_env="dev",
                            tracking_filename="tracking.log",
                            dev_env=True,
                            debug=True)

modulestore_options = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'fs_root': GITHUB_REPO_ROOT,
    'render_template': 'edxmako.shortcuts.render_to_string',
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': modulestore_options
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': modulestore_options
    },
    'split': {
        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': modulestore_options
    }
}


# cdodge: This is the specifier for the MongoDB (using GridFS) backed static content store
# This is for static content for courseware, not system static content (e.g. javascript, css, edX branding, etc)
CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'DOC_STORE_CONFIG': {
        'host': 'localhost',
        'db': 'xcontent',
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': ENV_ROOT / "db" / "edx.db",
    }
}

LMS_BASE = "localhost:8000"
FEATURES['PREVIEW_LMS_BASE'] = "localhost:8000"

REPOS = {
    'edx4edx': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/edx4edx.git',
    },
    'content-mit-6002x': {
        'branch': 'master',
        # 'origin': 'git@github.com:MITx/6002x-fall-2012.git',
        'origin': 'git@github.com:MITx/content-mit-6002x.git',
    },
    '6.00x': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/6.00x.git',
    },
    '7.00x': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/7.00x.git',
    },
    '3.091x': {
        'branch': 'master',
        'origin': 'git@github.com:MITx/3.091x.git',
    },
}

CACHES = {
    # This is the cache used for most things. Askbot will not work without a
    # functioning cache -- it relies on caching to load its settings in places.
    # In staging/prod envs, the sessions also live here.
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_loc_mem_cache',
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    # The general cache is what you get if you use our util.cache. It's used for
    # things like caching the course.xml file for different A/B test groups.
    # We set it to be a DummyCache to force reloading of course.xml in dev.
    # In staging environments, we would grab VERSION from data uploaded by the
    # push process.
    'general': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'KEY_PREFIX': 'general',
        'VERSION': 4,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    'mongo_metadata_inheritance': {
        'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
        'LOCATION': '/var/tmp/mongo_metadata_inheritance',
        'TIMEOUT': 300,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },
    'loc_cache': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_location_mem_cache',
    },

}

# Make the keyedcache startup warnings go away
CACHE_TIMEOUT = 0

# Dummy secret key for dev
SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'

################################ PIPELINE #################################

PIPELINE_SASS_ARGUMENTS = '--debug-info --require {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT)

################################# CELERY ######################################

# By default don't use a worker, execute tasks as if they were local functions
CELERY_ALWAYS_EAGER = True

################################ DEBUG TOOLBAR #################################
INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
INTERNAL_IPS = ('127.0.0.1',)

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',

    #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
    #  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
    #  hit twice). So you can uncomment when you need to diagnose performance
    #  problems, but you shouldn't leave it on.
    #  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
)

DEBUG_TOOLBAR_CONFIG = {
    'INTERCEPT_REDIRECTS': False
}

# To see stacktraces for MongoDB queries, set this to True.
# Stacktraces slow down page loads drastically (for pages with lots of queries).
DEBUG_TOOLBAR_MONGO_STACKTRACES = False

# disable NPS survey in dev mode
FEATURES['STUDIO_NPS_SURVEY'] = False

# Enable URL that shows information about the status of variuous services
FEATURES['ENABLE_SERVICE_STATUS'] = True

############################# SEGMENT-IO ##################################

# If there's an environment variable set, grab it and turn on Segment.io
# Note that this is the Studio key. There is a separate key for the LMS.
import os
SEGMENT_IO_KEY = os.environ.get('SEGMENT_IO_KEY')
if SEGMENT_IO_KEY:
    FEATURES['SEGMENT_IO'] = True


#####################################################################
# Lastly, see if the developer has any local overrides.
try:
    from .private import *  # pylint: disable=F0401
except ImportError:
    pass

########NEW FILE########
__FILENAME__ = devstack
"""
Specific overrides to the base prod settings to make development easier.
"""

from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import

# Don't use S3 in devstack, fall back to filesystem
del DEFAULT_FILE_STORAGE
MEDIA_ROOT = "/edx/var/edxapp/uploads"

DEBUG = True
USE_I18N = True
TEMPLATE_DEBUG = DEBUG

################################ LOGGERS ######################################

import logging

# Disable noisy loggers
for pkg_name in ['track.contexts', 'track.middleware', 'dd.dogapi']:
    logging.getLogger(pkg_name).setLevel(logging.CRITICAL)


################################ EMAIL ########################################

EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'

################################# LMS INTEGRATION #############################

LMS_BASE = "localhost:8000"
FEATURES['PREVIEW_LMS_BASE'] = "preview." + LMS_BASE

############################# ADVANCED COMPONENTS #############################

# Make it easier to test advanced components in local dev
FEATURES['ALLOW_ALL_ADVANCED_COMPONENTS'] = True

################################# CELERY ######################################

# By default don't use a worker, execute tasks as if they were local functions
CELERY_ALWAYS_EAGER = True

################################ DEBUG TOOLBAR ################################
INSTALLED_APPS += ('debug_toolbar', 'debug_toolbar_mongo')
MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',)
INTERNAL_IPS = ('127.0.0.1',)

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',

    #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
    #  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
    #  hit twice). So you can uncomment when you need to diagnose performance
    #  problems, but you shouldn't leave it on.
    #  'debug_toolbar.panels.profiling.ProfilingPanel',
)

DEBUG_TOOLBAR_CONFIG = {
    'INTERCEPT_REDIRECTS': False,
    'SHOW_TOOLBAR_CALLBACK': lambda _: True,
}

# To see stacktraces for MongoDB queries, set this to True.
# Stacktraces slow down page loads drastically (for pages with lots of queries).
DEBUG_TOOLBAR_MONGO_STACKTRACES = False

###############################################################################
# Lastly, see if the developer has any local overrides.
try:
    from .private import *  # pylint: disable=F0401
except ImportError:
    pass

########NEW FILE########
__FILENAME__ = dev_dbperf
"""
This configuration is to turn on the Django Toolbar stats for DB access stats, for performance analysis
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .dev import *

DEBUG_TOOLBAR_PANELS = (
    'debug_toolbar.panels.version.VersionDebugPanel',
    'debug_toolbar.panels.timer.TimerDebugPanel',
    'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel',
    'debug_toolbar.panels.headers.HeaderDebugPanel',
    'debug_toolbar.panels.request_vars.RequestVarsDebugPanel',
    'debug_toolbar.panels.sql.SQLDebugPanel',
    'debug_toolbar.panels.signals.SignalDebugPanel',
    'debug_toolbar.panels.logger.LoggingPanel',
    'debug_toolbar_mongo.panel.MongoDebugPanel'

    #  Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and
    #  Django=1.3.1/1.4 where requests to views get duplicated (your method gets
    #  hit twice). So you can uncomment when you need to diagnose performance
    #  problems, but you shouldn't leave it on.
    #  'debug_toolbar.panels.profiling.ProfilingDebugPanel',
)

# To see stacktraces for MongoDB queries, set this to True.
# Stacktraces slow down page loads drastically (for pages with lots of queries).
DEBUG_TOOLBAR_MONGO_STACKTRACES = True

########NEW FILE########
__FILENAME__ = dev_ike
# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

# dev environment for ichuang/mit

# FORCE_SCRIPT_NAME = '/cms'

from .common import *
from .dev import *

FEATURES['AUTH_USE_CERTIFICATES'] = True

FEATURES['USE_DJANGO_PIPELINE'] = False      # don't recompile scss

SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')  	# django 1.4 for nginx ssl proxy

########NEW FILE########
__FILENAME__ = dev_shared_preview
"""
This configuration is have localdev use a preview.localhost hostname for the preview LMS so that we can share
the same process between preview and published
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .dev import *

FEATURES['PREVIEW_LMS_BASE'] = "preview.localhost:8000"

########NEW FILE########
__FILENAME__ = dev_with_worker
"""
This config file follows the dev enviroment, but adds the
requirement of a celery worker running in the background to process
celery tasks.

The worker can be executed using:

django_admin.py celery worker
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from dev import *

################################# CELERY ######################################

# Requires a separate celery worker

CELERY_ALWAYS_EAGER = False

# Use django db as the broker and result store

BROKER_URL = 'django://'
INSTALLED_APPS += ('djcelery.transport', )
CELERY_RESULT_BACKEND = 'database'
DJKOMBU_POLLING_INTERVAL = 1.0

# Disable transaction management because we are using a worker. Views
# that request a task and wait for the result will deadlock otherwise.

MIDDLEWARE_CLASSES = tuple(
    c for c in MIDDLEWARE_CLASSES
    if c != 'django.middleware.transaction.TransactionMiddleware')

# Note: other alternatives for disabling transactions don't work in 1.4
# https://code.djangoproject.com/ticket/2304
# https://code.djangoproject.com/ticket/16039

########NEW FILE########
__FILENAME__ = microsite_test
"""
This is a localdev test for the Microsite processing pipeline
"""
# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .dev import *

MICROSITE_NAMES = ['openedx']
MICROSITE_CONFIGURATION = {}
FEATURES['USE_MICROSITES'] = True

########NEW FILE########
__FILENAME__ = test
"""
This config file runs the simplest dev environment using sqlite, and db-based
sessions. Assumes structure:

/envroot/
        /db   # This is where it'll write the database file
        /edx-platform  # The location of this repo
        /log  # Where we're going to write log files
"""

# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=W0401, W0614

from .common import *
import os
from path import path
from warnings import filterwarnings

# import settings from LMS for consistent behavior with CMS
from lms.envs.test import (WIKI_ENABLED)

# Nose Test Runner
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'

_system = 'cms'
_report_dir = REPO_ROOT / 'reports' / _system
_report_dir.makedirs_p()

NOSE_ARGS = [
    '--id-file', REPO_ROOT / '.testids' / _system / 'noseids',
    '--xunit-file', _report_dir / 'nosetests.xml',
]

TEST_ROOT = path('test_root')

# Want static files in the same dir for running on jenkins.
STATIC_ROOT = TEST_ROOT / "staticfiles"

GITHUB_REPO_ROOT = TEST_ROOT / "data"
COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"

# For testing "push to lms"
FEATURES['ENABLE_EXPORT_GIT'] = True
GIT_REPO_EXPORT_DIR = TEST_ROOT / "export_course_repos"

# Makes the tests run much faster...
SOUTH_TESTS_MIGRATE = False  # To disable migrations and use syncdb instead

# TODO (cpennington): We need to figure out how envs/test.py can inject things into common.py so that we don't have to repeat this sort of thing
STATICFILES_DIRS = [
    COMMON_ROOT / "static",
    PROJECT_ROOT / "static",
]
STATICFILES_DIRS += [
    (course_dir, COMMON_TEST_DATA_ROOT / course_dir)
    for course_dir in os.listdir(COMMON_TEST_DATA_ROOT)
    if os.path.isdir(COMMON_TEST_DATA_ROOT / course_dir)
]

DOC_STORE_CONFIG = {
    'host': 'localhost',
    'db': 'test_xmodule',
    'collection': 'test_modulestore',
}

MODULESTORE_OPTIONS = {
    'default_class': 'xmodule.raw_module.RawDescriptor',
    'fs_root': TEST_ROOT / "data",
    'render_template': 'edxmako.shortcuts.render_to_string',
}

MODULESTORE = {
    'default': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'direct': {
        'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'draft': {
        'ENGINE': 'xmodule.modulestore.draft.DraftModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    },
    'split': {
        'ENGINE': 'xmodule.modulestore.split_mongo.SplitMongoModuleStore',
        'DOC_STORE_CONFIG': DOC_STORE_CONFIG,
        'OPTIONS': MODULESTORE_OPTIONS
    }
}

CONTENTSTORE = {
    'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
    'DOC_STORE_CONFIG': {
        'host': 'localhost',
        'db': 'test_xcontent',
        'collection': 'dont_trip',
    },
    # allow for additional options that can be keyed on a name, e.g. 'trashcan'
    'ADDITIONAL_OPTIONS': {
        'trashcan': {
            'bucket': 'trash_fs'
        }
    }
}

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': TEST_ROOT / "db" / "cms.db",
    },
}

LMS_BASE = "localhost:8000"
FEATURES['PREVIEW_LMS_BASE'] = "preview"

CACHES = {
    # This is the cache used for most things. Askbot will not work without a
    # functioning cache -- it relies on caching to load its settings in places.
    # In staging/prod envs, the sessions also live here.
    'default': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_loc_mem_cache',
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    # The general cache is what you get if you use our util.cache. It's used for
    # things like caching the course.xml file for different A/B test groups.
    # We set it to be a DummyCache to force reloading of course.xml in dev.
    # In staging environments, we would grab VERSION from data uploaded by the
    # push process.
    'general': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'KEY_PREFIX': 'general',
        'VERSION': 4,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },

    'mongo_metadata_inheritance': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': '/var/tmp/mongo_metadata_inheritance',
        'TIMEOUT': 300,
        'KEY_FUNCTION': 'util.memcache.safe_key',
    },
    'loc_cache': {
        'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
        'LOCATION': 'edx_location_mem_cache',
    },

}

# Add external_auth to Installed apps for testing
INSTALLED_APPS += ('external_auth', )

# hide ratelimit warnings while running tests
filterwarnings('ignore', message='No request passed to the backend, unable to rate-limit')


################################# CELERY ######################################

CELERY_ALWAYS_EAGER = True
CELERY_RESULT_BACKEND = 'cache'
BROKER_TRANSPORT = 'memory'


########################### Server Ports ###################################

# These ports are carefully chosen so that if the browser needs to
# access them, they will be available through the SauceLabs SSH tunnel
LETTUCE_SERVER_PORT = 8003
XQUEUE_PORT = 8040
YOUTUBE_PORT = 8031
LTI_PORT = 8765
VIDEO_SOURCE_PORT = 8777


################### Make tests faster
# http://slacy.com/blog/2012/04/make-your-tests-faster-in-django-1-4/
PASSWORD_HASHERS = (
    'django.contrib.auth.hashers.SHA1PasswordHasher',
    'django.contrib.auth.hashers.MD5PasswordHasher',
)

# dummy segment-io key
SEGMENT_IO_KEY = '***REMOVED***'

# disable NPS survey in test mode
FEATURES['STUDIO_NPS_SURVEY'] = False

FEATURES['ENABLE_SERVICE_STATUS'] = True

# This is to disable a test under the common directory that will not pass when run under CMS
FEATURES['DISABLE_RESET_EMAIL_TEST'] = True

# Toggles embargo on for testing
FEATURES['EMBARGO'] = True

# set up some testing for microsites
MICROSITE_CONFIGURATION = {
    "test_microsite": {
        "domain_prefix": "testmicrosite",
        "university": "test_microsite",
        "platform_name": "Test Microsite",
        "logo_image_url": "test_microsite/images/header-logo.png",
        "email_from_address": "test_microsite@edx.org",
        "payment_support_email": "test_microsite@edx.org",
        "ENABLE_MKTG_SITE": False,
        "SITE_NAME": "test_microsite.localhost",
        "course_org_filter": "TestMicrositeX",
        "course_about_show_social_links": False,
        "css_overrides_file": "test_microsite/css/test_microsite.css",
        "show_partners": False,
        "show_homepage_promo_video": False,
        "course_index_overlay_text": "This is a Test Microsite Overlay Text.",
        "course_index_overlay_logo_file": "test_microsite/images/header-logo.png",
        "homepage_overlay_html": "<h1>This is a Test Microsite Overlay HTML</h1>"
    },
    "default": {
        "university": "default_university",
        "domain_prefix": "www",
    }
}
MICROSITE_ROOT_DIR = COMMON_ROOT / 'test' / 'test_microsites'
FEATURES['USE_MICROSITES'] = True

# For consistency in user-experience, keep the value of this setting in sync with
# the one in lms/envs/test.py
FEATURES['ENABLE_DISCUSSION_SERVICE'] = False

########NEW FILE########
__FILENAME__ = mixin
"""
Mixin defining common Studio functionality
"""

import datetime

from xblock.fields import Scope, Field, Integer, XBlockMixin


class DateTuple(Field):
    """
    Field that stores datetime objects as time tuples
    """
    def from_json(self, value):
        return datetime.datetime(*value[0:6])

    def to_json(self, value):
        if value is None:
            return None

        return list(value.timetuple())


class CmsBlockMixin(XBlockMixin):
    """
    Mixin with fields common to all blocks in Studio
    """
    published_date = DateTuple(help="Date when the module was published", scope=Scope.settings)
    published_by = Integer(help="Id of the user who published this module", scope=Scope.settings)

########NEW FILE########
__FILENAME__ = runtime
"""
XBlock runtime implementations for edX Studio
"""

from django.core.urlresolvers import reverse


def handler_url(block, handler_name, suffix='', query='', thirdparty=False):
    """
    Handler URL function for Studio
    """

    if thirdparty:
        raise NotImplementedError("edX Studio doesn't support third-party xblock handler urls")

    url = reverse('component_handler', kwargs={
        'usage_key_string': unicode(block.scope_ids.usage_id).encode('utf-8'),
        'handler': handler_name,
        'suffix': suffix,
    }).rstrip('/')

    if query:
        url += '?' + query

    return url


def local_resource_url(block, uri):
    """
    local_resource_url for Studio
    """
    return reverse('xblock_resource_url', kwargs={
        'block_type': block.scope_ids.block_type,
        'uri': uri,
    })

########NEW FILE########
__FILENAME__ = test_runtime
"""
Tests of edX Studio runtime functionality
"""
from urlparse import urlparse

from mock import Mock
from unittest import TestCase
from cms.lib.xblock.runtime import handler_url


class TestHandlerUrl(TestCase):
    """Test the LMS handler_url"""

    def setUp(self):
        self.block = Mock()

    def test_trailing_charecters(self):
        self.assertFalse(handler_url(self.block, 'handler').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler').endswith('/'))

        self.assertFalse(handler_url(self.block, 'handler', 'suffix').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler', 'suffix').endswith('/'))

        self.assertFalse(handler_url(self.block, 'handler', 'suffix', 'query').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler', 'suffix', 'query').endswith('/'))

        self.assertFalse(handler_url(self.block, 'handler', query='query').endswith('?'))
        self.assertFalse(handler_url(self.block, 'handler', query='query').endswith('/'))

    def _parsed_query(self, query_string):
        """Return the parsed query string from a handler_url generated with the supplied query_string"""
        return urlparse(handler_url(self.block, 'handler', query=query_string)).query

    def test_query_string(self):
        self.assertIn('foo=bar', self._parsed_query('foo=bar'))
        self.assertIn('foo=bar&baz=true', self._parsed_query('foo=bar&baz=true'))
        self.assertIn('foo&bar&baz', self._parsed_query('foo&bar&baz'))

    def _parsed_path(self, handler_name='handler', suffix=''):
        """Return the parsed path from a handler_url with the supplied handler_name and suffix"""
        return urlparse(handler_url(self.block, handler_name, suffix=suffix)).path

    def test_suffix(self):
        self.assertTrue(self._parsed_path(suffix="foo").endswith('foo'))
        self.assertTrue(self._parsed_path(suffix="foo/bar").endswith('foo/bar'))
        self.assertTrue(self._parsed_path(suffix="/foo/bar").endswith('/foo/bar'))

    def test_handler_name(self):
        self.assertIn('handler1', self._parsed_path('handler1'))
        self.assertIn('handler_a', self._parsed_path('handler_a'))

########NEW FILE########
__FILENAME__ = startup
"""
Module with code executed during Studio startup
"""
from django.conf import settings

# Force settings to run so that the python path is modified
settings.INSTALLED_APPS  # pylint: disable=W0104

from django_startup import autostartup


def run():
    """
    Executed during django startup
    """
    autostartup()

########NEW FILE########
__FILENAME__ = urls
from django.conf import settings
from django.conf.urls import patterns, include, url

# There is a course creators admin table.
from ratelimitbackend import admin
admin.autodiscover()

urlpatterns = patterns('',  # nopep8

    url(r'^transcripts/upload$', 'contentstore.views.upload_transcripts', name='upload_transcripts'),
    url(r'^transcripts/download$', 'contentstore.views.download_transcripts', name='download_transcripts'),
    url(r'^transcripts/check$', 'contentstore.views.check_transcripts', name='check_transcripts'),
    url(r'^transcripts/choose$', 'contentstore.views.choose_transcripts', name='choose_transcripts'),
    url(r'^transcripts/replace$', 'contentstore.views.replace_transcripts', name='replace_transcripts'),
    url(r'^transcripts/rename$', 'contentstore.views.rename_transcripts', name='rename_transcripts'),
    url(r'^transcripts/save$', 'contentstore.views.save_transcripts', name='save_transcripts'),

    url(r'^preview/xblock/(?P<usage_key_string>.*?)/handler/(?P<handler>[^/]*)(?:/(?P<suffix>.*))?$',
        'contentstore.views.preview_handler', name='preview_handler'),

    url(r'^xblock/(?P<usage_key_string>.*?)/handler/(?P<handler>[^/]*)(?:/(?P<suffix>.*))?$',
        'contentstore.views.component_handler', name='component_handler'),

    url(r'^xblock/resource/(?P<block_type>[^/]*)/(?P<uri>.*)$',
        'contentstore.views.xblock.xblock_resource', name='xblock_resource_url'),

    # temporary landing page for a course
    url(r'^edge/(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<coursename>[^/]+)$',
        'contentstore.views.landing', name='landing'),

    url(r'^not_found$', 'contentstore.views.not_found', name='not_found'),
    url(r'^server_error$', 'contentstore.views.server_error', name='server_error'),

    # temporary landing page for edge
    url(r'^edge$', 'contentstore.views.edge', name='edge'),
    # noop to squelch ajax errors
    url(r'^event$', 'contentstore.views.event', name='event'),

    url(r'^xmodule/', include('pipeline_js.urls')),
    url(r'^heartbeat$', include('heartbeat.urls')),

    url(r'^user_api/', include('user_api.urls')),
    url(r'^lang_pref/', include('lang_pref.urls')),
)

# User creation and updating views
urlpatterns += patterns(
    '',

    url(r'^create_account$', 'student.views.create_account', name='create_account'),
    url(r'^activate/(?P<key>[^/]*)$', 'student.views.activate_account', name='activate'),

    # ajax view that actually does the work
    url(r'^login_post$', 'student.views.login_user', name='login_post'),
    url(r'^logout$', 'student.views.logout_user', name='logout'),
    url(r'^embargo$', 'student.views.embargo', name="embargo"),
)

# restful api
urlpatterns += patterns(
    'contentstore.views',

    url(r'^$', 'howitworks', name='homepage'),
    url(r'^howitworks$', 'howitworks'),
    url(r'^signup$', 'signup', name='signup'),
    url(r'^signin$', 'login_page', name='login'),
    url(r'^request_course_creator$', 'request_course_creator'),

    url(r'^course_team/(?P<course_key_string>[^/]+)/(?P<email>.+)?$', 'course_team_handler'),
    url(r'^course_info/(?P<course_key_string>[^/]+)$', 'course_info_handler'),
    url(
        r'^course_info_update/(?P<course_key_string>[^/]+)/(?P<provided_id>\d+)?$',
        'course_info_update_handler'
    ),
    url(r'^course/(?P<course_key_string>[^/]+)?$', 'course_handler', name='course_handler'),
    url(r'^subsection/(?P<usage_key_string>[^/]+)$', 'subsection_handler'),
    url(r'^unit/(?P<usage_key_string>[^/]+)$', 'unit_handler'),
    url(r'^container/(?P<usage_key_string>[^/]+)$', 'container_handler'),
    url(r'^checklists/(?P<course_key_string>[^/]+)/(?P<checklist_index>\d+)?$', 'checklists_handler'),
    url(r'^orphan/(?P<course_key_string>[^/]+)$', 'orphan_handler'),
    url(r'^assets/(?P<course_key_string>[^/]+)/(?P<asset_key_string>.+)?$', 'assets_handler'),
    url(r'^import/(?P<course_key_string>[^/]+)$', 'import_handler'),
    url(r'^import_status/(?P<course_key_string>[^/]+)/(?P<filename>.+)$', 'import_status_handler'),
    url(r'^export/(?P<course_key_string>[^/]+)$', 'export_handler'),
    url(r'^xblock/(?P<usage_key_string>[^/]+)/(?P<view_name>[^/]+)$', 'xblock_view_handler'),
    url(r'^xblock/(?P<usage_key_string>[^/]+)?$', 'xblock_handler'),
    url(r'^tabs/(?P<course_key_string>[^/]+)$', 'tabs_handler'),
    url(r'^settings/details/(?P<course_key_string>[^/]+)$', 'settings_handler'),
    url(r'^settings/grading/(?P<course_key_string>[^/]+)(/)?(?P<grader_index>\d+)?$', 'grading_handler'),
    url(r'^settings/advanced/(?P<course_key_string>[^/]+)$', 'advanced_settings_handler'),
    url(r'^textbooks/(?P<course_key_string>[^/]+)$', 'textbooks_list_handler'),
    url(r'^textbooks/(?P<course_key_string>[^/]+)/(?P<textbook_id>\d[^/]*)$', 'textbooks_detail_handler'),
)

js_info_dict = {
    'domain': 'djangojs',
    # No packages needed, we get LOCALE_PATHS anyway.
    'packages': (),
}

urlpatterns += patterns('',
    # Serve catalog of localized strings to be rendered by Javascript
    url(r'^i18n.js$', 'django.views.i18n.javascript_catalog', js_info_dict),
)


if settings.FEATURES.get('ENABLE_EXPORT_GIT'):
    urlpatterns += (url(r'^export_git/(?P<course_key_string>[^/]+)$',
                        'contentstore.views.export_git', name='export_git'),)

if settings.FEATURES.get('ENABLE_SERVICE_STATUS'):
    urlpatterns += patterns('',
        url(r'^status/', include('service_status.urls')),
    )

if settings.FEATURES.get('AUTH_USE_CAS'):
    urlpatterns += (
        url(r'^cas-auth/login/$', 'external_auth.views.cas_login', name="cas-login"),
        url(r'^cas-auth/logout/$', 'django_cas.views.logout', {'next_page': '/'}, name="cas-logout"),
    )

urlpatterns += patterns('', url(r'^admin/', include(admin.site.urls)),)

# enable automatic login
if settings.FEATURES.get('AUTOMATIC_AUTH_FOR_TESTING'):
    urlpatterns += (
        url(r'^auto_auth$', 'student.views.auto_auth'),
    )

if settings.DEBUG:
    try:
        from .urls_dev import urlpatterns as dev_urlpatterns
        urlpatterns += dev_urlpatterns
    except ImportError:
        pass

# Custom error pages
# pylint: disable=C0103
handler404 = 'contentstore.views.render_404'
handler500 = 'contentstore.views.render_500'

# display error page templates, for testing purposes
urlpatterns += (
    url(r'404', handler404),
    url(r'500', handler500),
)

########NEW FILE########
__FILENAME__ = urls_dev
"""
URLconf for development-only views.
This gets imported by urls.py and added to its URLconf if we are running in
development mode; otherwise, it is ignored.
"""
from django.conf.urls import url

urlpatterns = (
    url(r'^dev_mode$', 'contentstore.views.dev.dev_mode', name='dev_mode'),
    url(r'^template/(?P<template>.+)$', 'contentstore.views.dev.dev_show_template'),
)

########NEW FILE########
__FILENAME__ = wsgi
import os

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cms.envs.aws")

import cms.startup as startup
startup.run()

# This application object is used by the development server
# as well as any WSGI server configured to use this file.
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()


########NEW FILE########
__FILENAME__ = app_settings
from django.conf import settings

# Default cache timeout
CACHE_TOOLBOX_DEFAULT_TIMEOUT = getattr(
    settings,
    'CACHE_TOOLBOX_DEFAULT_TIMEOUT',
    60 * 60 * 24 * 3,
)

########NEW FILE########
__FILENAME__ = core
"""
Core methods
------------

.. autofunction:: cache_toolbox.core.get_instance
.. autofunction:: cache_toolbox.core.delete_instance
.. autofunction:: cache_toolbox.core.instance_key

"""

from django.core.cache import cache
from django.db import DEFAULT_DB_ALIAS

from . import app_settings


def get_instance(model, instance_or_pk, timeout=None, using=None):
    """
    Returns the ``model`` instance with a primary key of ``instance_or_pk``.

    If the data is cached it will be returned from there, otherwise the regular
    Django ORM is queried for this instance and the data stored in the cache.

    If omitted, the timeout value defaults to
    ``settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT`` instead of 0 (zero).

    Example::

        >>> get_instance(User, 1) # Cache miss
        <User: lamby>
        >>> get_instance(User, 1) # Cache hit
        <User: lamby>
        >>> User.objects.get(pk=1) == get_instance(User, 1)
        True
    """

    pk = getattr(instance_or_pk, 'pk', instance_or_pk)
    key = instance_key(model, instance_or_pk)
    data = cache.get(key)

    if data is not None:
        try:
            # Try and construct instance from dictionary
            instance = model(pk=pk, **data)

            # Ensure instance knows that it already exists in the database,
            # otherwise we will fail any uniqueness checks when saving the
            # instance.
            instance._state.adding = False

            # Specify database so that instance is setup correctly. We don't
            # namespace cached objects by their origin database, however.
            instance._state.db = using or DEFAULT_DB_ALIAS

            return instance
        except:
            # Error when deserialising - remove from the cache; we will
            # fallback and return the underlying instance
            cache.delete(key)

    # Use the default manager so we are never filtered by a .get_query_set()

#    import logging
#    log = logging.getLogger("tracking")
#    log.info( str(pk) )

    instance = model._default_manager.using(using).get(pk=pk)

    data = {}
    for field in instance._meta.fields:
        # Harmless to save, but saves space in the dictionary - we already know
        # the primary key when we lookup
        if field.primary_key:
            continue

        if field.get_internal_type() == 'FileField':
            # Avoid problems with serializing FileFields
            # by only serializing the file name
            file = getattr(instance, field.attname)
            data[field.attname] = file.name
        else:
            data[field.attname] = getattr(instance, field.attname)

    if timeout is None:
        timeout = app_settings.CACHE_TOOLBOX_DEFAULT_TIMEOUT

    cache.set(key, data, timeout)

    return instance


def delete_instance(model, *instance_or_pk):
    """
    Purges the cache keys for the instances of this model.
    """

    cache.delete_many([instance_key(model, x) for x in instance_or_pk])


def instance_key(model, instance_or_pk):
    """
    Returns the cache key for this (model, instance) pair.
    """

    return '%s.%s:%d' % (
        model._meta.app_label,
        model._meta.module_name,
        getattr(instance_or_pk, 'pk', instance_or_pk),
    )


def set_cached_content(content):
    cache.set(unicode(content.location).encode("utf-8"), content)


def get_cached_content(location):
    return cache.get(unicode(location).encode("utf-8"))


def del_cached_content(location):
    # delete content for the given location, as well as for content with run=None.
    # it's possible that the content could have been cached without knowing the
    # course_key - and so without having the run.
    cache.delete_many(
        [unicode(loc).encode("utf-8") for loc in [location, location.replace(run=None)]]
    )

########NEW FILE########
__FILENAME__ = middleware
"""
Cache-backed ``AuthenticationMiddleware``
-----------------------------------------

``CacheBackedAuthenticationMiddleware`` is an
``django.contrib.auth.middleware.AuthenticationMiddleware`` replacement to
avoid querying the database for a ``User`` instance in each request.

Whilst the built-in ``AuthenticationMiddleware`` mechanism will only obtain the
``User`` instance when it is required, the vast majority of sites will do so on
every page to render "Logged in as 'X'" text as well to evaluate the result of
``user.is_authenticated()`` and ``user.is_superuser`` to provide conditional
functionality.

This middleware eliminates the cost of retrieving this ``User`` instance by
caching it using the ``cache_toolbox`` instance caching mechanisms.

Depending on your average number of queries per page, saving one query per
request can---in aggregate---reduce load on your database. In addition,
avoiding the database entirely for pages can avoid incurring any connection
latency in your environment, resulting in faster page loads for your users.

Saving this data in the cache can also be used as a way of authenticating users
in systems outside of Django that should not access your database.  For
example, a "maintenance mode" page would be able to render a personalised
message without touching the database at all but rather authenticating via the
cache.

``CacheBackedAuthenticationMiddleware`` is ``AUTHENTICATION_BACKENDS`` agnostic.

Implementation
~~~~~~~~~~~~~~

The cache and session backends are still accessed on each request - we are
simply assuming that they are cheaper (or otherwise more preferable) to access
than your database. (In the future, signed cookies may allow us to avoid this
lookup altogether -- whilst we could not safely save ``User.password`` in a
cookie, we could use delayed loading to pull it out when needed.)

Another alternative solution would be to store the attributes in the user's
session instead of in the cache. This would save the cache hit on every request
as all the relevant data would be pulled in one go from the session backend.
However, this has two main disadvantages:

 * Session keys are not deterministic -- after making changes to an
   ``auth_user`` row in the database, you cannot determine the user's session
   key to flush the now out-of-sync data (and doing so would log them out
   anyway).

 * Stores data per-session rather than per-user -- if a user logs in from
   multiple computers the data is duplicated in each session. This problem is
   compounded by most projects wishing to avoid expiring session data as long
   as possible (in addition to storing sessions in persistent stores).

Usage
~~~~~

To use, find ``MIDDLEWARE_CLASSES`` in your ``settings.py`` and replace::

    MIDDLEWARE_CLASSES = [
        ...
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        ...
    ]

with::

    MIDDLEWARE_CLASSES = [
        ...
        'cache_toolbox.middleware.CacheBackedAuthenticationMiddleware',
        ...
    ]

You should confirm you are using a ``SESSION_ENGINE`` that doesn't query the
database for each request. The built-in ``cached_db`` engine is the safest
choice for most environments but you may be happy with the trade-offs of the
``memcached`` backend - see the Django documentation for more details.

"""

from django.contrib.auth import SESSION_KEY
from django.contrib.auth.models import User
from django.contrib.auth.middleware import AuthenticationMiddleware

from .model import cache_model


class CacheBackedAuthenticationMiddleware(AuthenticationMiddleware):
    def __init__(self):
        cache_model(User)

    def process_request(self, request):
        try:
            # Try and construct a User instance from data stored in the cache
            request.user = User.get_cached(request.session[SESSION_KEY])
        except:
            # Fallback to constructing the User from the database.
            super(CacheBackedAuthenticationMiddleware, self).process_request(request)

########NEW FILE########
__FILENAME__ = model
"""
Caching model instances
-----------------------

``cache_model`` adds utility methods to a model to obtain ``ForeignKey``
instances via the cache.

Usage
~~~~~

::

    from django.db import models
    from django.contrib.auth.models import User

    class Foo(models.Model):
        name = models.CharField(length=20)

    cache_model(Foo)

::

    >>> a = Foo.objects.create(name='a')
    >>> a
    <Foo: >
    >>> Foo.get_cached(a.pk) # Cache miss
    <Foo: >
    >>> a = Foo.get_cached(a.pk) # Cache hit
    >>> a.name
    u'a'

Instances returned from ``get_cached`` are real model instances::

    >>> a = Foo.get_cached(a.pk) # Cache hit
    >>> type(a)
    <class '__main__.models.A'>
    >>> a.pk
    1L

Invalidation
~~~~~~~~~~~~

Invalidation is performed automatically upon saving or deleting a ``Foo``
instance::

    >>> a = Foo.objects.create(name='a')
    >>> a.name = 'b'
    >>> a.save()
    >>> a = Foo.get_cached(a.pk)
    >>> a.name
    u'b'
    >>> a.delete()
    >>> a = Foo.get_cached(a.pk)
    ... Foo.DoesNotExist
"""

from django.db.models.signals import post_save, post_delete

from .core import get_instance, delete_instance


def cache_model(model, timeout=None):
    if hasattr(model, 'get_cached'):
        # Already patched
        return

    def clear_cache(sender, instance, *args, **kwargs):
        delete_instance(sender, instance)

    post_save.connect(clear_cache, sender=model, weak=False)
    post_delete.connect(clear_cache, sender=model, weak=False)

    @classmethod
    def get(cls, pk, using=None):
        if pk is None:
            return None
        return get_instance(cls, pk, timeout, using)

    model.get_cached = get

########NEW FILE########
__FILENAME__ = relation
"""
Caching instances via ``related_name``
--------------------------------------

``cache_relation`` adds utility methods to a model to obtain ``related_name``
instances via the cache.

Usage
~~~~~

::

    from django.db import models
    from django.contrib.auth.models import User

    class Foo(models.Model):
        user = models.OneToOneField(
            User,
            primary_key=True,
            related_name='foo',
        )

        name = models.CharField(max_length=20)

    cache_relation(User.foo)

::

    >>> user = User.objects.get(pk=1)
    >>> user.foo_cache # Cache miss - hits the database
    <Foo: >
    >>> user = User.objects.get(pk=1)
    >>> user.foo_cache # Cache hit - no database access
    <Foo: >
    >>> user = User.objects.get(pk=2)
    >>> user.foo # Regular lookup - hits the database
    <Foo: >
    >>> user.foo_cache # Special-case: Will not hit cache or database.
    <Foo: >

Accessing ``user_instance.foo_cache`` (note the "_cache" suffix) will now
obtain the related ``Foo`` instance via the cache. Accessing the original
``user_instance.foo`` attribute will perform the lookup as normal.

Invalidation
~~~~~~~~~~~~

Upon saving (or deleting) the instance, the cache is cleared. For example::

    >>> user = User.objects.get(pk=1)
    >>> foo = user.foo_cache # (Assume cache hit from previous session)
    >>> foo.name = "New name"
    >>> foo.save() # Cache is cleared on save
    >>> user = User.objects.get(pk=1)
    >>> user.foo_cache # Cache miss.
    <Foo: >

Manual invalidation may also be performed using the following methods::

    >>> user_instance.foo_cache_clear()
    >>> User.foo_cache_clear_fk(user_instance_pk)

Manual invalidation is required if you use ``.update()`` methods which the
``post_save`` and ``post_delete`` hooks cannot intercept.

Support
~~~~~~~

``cache_relation`` currently only works with ``OneToOneField`` fields. Support
for regular ``ForeignKey`` fields is planned.
"""

from django.db.models.signals import post_save, post_delete

from .core import get_instance, delete_instance


def cache_relation(descriptor, timeout=None):
    rel = descriptor.related
    related_name = '%s_cache' % rel.field.related_query_name()

    @property
    def get(self):
        # Always use the cached "real" instance if available
        try:
            return getattr(self, descriptor.cache_name)
        except AttributeError:
            pass

        # Lookup cached instance
        try:
            return getattr(self, '_%s_cache' % related_name)
        except AttributeError:
            pass

#        import logging
#        log = logging.getLogger("tracking")
#        log.info( "DEBUG: "+str(str(rel.model)+"/"+str(self.pk) ))

        instance = get_instance(rel.model, self.pk, timeout)

        setattr(self, '_%s_cache' % related_name, instance)

        return instance
    setattr(rel.parent_model, related_name, get)

    # Clearing cache

    def clear(self):
        delete_instance(rel.model, self)

    @classmethod
    def clear_pk(cls, *instances_or_pk):
        delete_instance(rel.model, *instances_or_pk)

    def clear_cache(sender, instance, *args, **kwargs):
        delete_instance(rel.model, instance)

    setattr(rel.parent_model, '%s_clear' % related_name, clear)
    setattr(rel.parent_model, '%s_clear_pk' % related_name, clear_pk)

    post_save.connect(clear_cache, sender=rel.model, weak=False)
    post_delete.connect(clear_cache, sender=rel.model, weak=False)

########NEW FILE########
__FILENAME__ = cache_toolbox
from django import template
from django.core.cache import cache
from django.template import Node, TemplateSyntaxError, Variable
from django.template import resolve_variable

register = template.Library()


class CacheNode(Node):
    def __init__(self, nodelist, expire_time, key):
        self.nodelist = nodelist
        self.expire_time = Variable(expire_time)
        self.key = key

    def render(self, context):
        key = resolve_variable(self.key, context)
        expire_time = int(self.expire_time.resolve(context))

        value = cache.get(key)
        if value is None:
            value = self.nodelist.render(context)
            cache.set(key, value, expire_time)
        return value


@register.tag
def cachedeterministic(parser, token):
    """
    This will cache the contents of a template fragment for a given amount of
    time, just like {% cache .. %} except that the key is deterministic and not
    mangled or run through MD5.

    Usage::

        {% cachedeterministic [expire_time] [key] %}
            .. some expensive processing ..
        {% endcachedeterministic %}

    """
    nodelist = parser.parse(('endcachedeterministic',))
    parser.delete_first_token()
    tokens = token.contents.split()
    if len(tokens) != 3:
        raise TemplateSyntaxError(u"'%r' tag requires 2 arguments." % tokens[0])
    return CacheNode(nodelist, tokens[1], tokens[2])


class ShowIfCachedNode(Node):
    def __init__(self, key):
        self.key = key

    def render(self, context):
        key = resolve_variable(self.key, context)
        return cache.get(key) or ''


@register.tag
def showifcached(parser, token):
    """
    Show content if it exists in the cache, otherwise display nothing.

    The key is entirely deterministic and not mangled or run through MD5 (cf.
    {% cache %})

    Usage::

        {% showifcached [key] %}

    """
    tokens = token.contents.split()
    if len(tokens) != 2:
        raise TemplateSyntaxError(u"'%r' tag requires 1 argument." % tokens[0])
    return ShowIfCachedNode(tokens[1])

########NEW FILE########
__FILENAME__ = admin
"""
Admin site models for managing :class:`.ConfigurationModel` subclasses
"""

from django.forms import models
from django.contrib import admin
from django.http import HttpResponseRedirect
from django.core.urlresolvers import reverse

# pylint: disable=protected-access


class ConfigurationModelAdmin(admin.ModelAdmin):
    """
    :class:`~django.contrib.admin.ModelAdmin` for :class:`.ConfigurationModel` subclasses
    """
    date_hierarchy = 'change_date'

    def get_actions(self, request):
        return {
            'revert': (ConfigurationModelAdmin.revert, 'revert', 'Revert to the selected configuration')
        }

    def get_list_display(self, request):
        return self.model._meta.get_all_field_names()

    # Don't allow deletion of configuration
    def has_delete_permission(self, request, obj=None):
        return False

    # Make all fields read-only when editing an object
    def get_readonly_fields(self, request, obj=None):
        if obj:  # editing an existing object
            return self.model._meta.get_all_field_names()
        return self.readonly_fields

    def add_view(self, request, form_url='', extra_context=None):
        # Prepopulate new configuration entries with the value of the current config
        get = request.GET.copy()
        get.update(models.model_to_dict(self.model.current()))
        request.GET = get
        return super(ConfigurationModelAdmin, self).add_view(request, form_url, extra_context)

    # Hide the save buttons in the change view
    def change_view(self, request, object_id, form_url='', extra_context=None):
        extra_context = extra_context or {}
        extra_context['readonly'] = True
        return super(ConfigurationModelAdmin, self).change_view(
            request,
            object_id,
            form_url,
            extra_context=extra_context
        )

    def save_model(self, request, obj, form, change):
        obj.changed_by = request.user
        super(ConfigurationModelAdmin, self).save_model(request, obj, form, change)

    def revert(self, request, queryset):
        """
        Admin action to revert a configuration back to the selected value
        """
        if queryset.count() != 1:
            self.message_user(request, "Please select a single configuration to revert to.")
            return

        target = queryset[0]
        target.id = None
        self.save_model(request, target, None, False)
        self.message_user(request, "Reverted configuration.")

        return HttpResponseRedirect(
            reverse(
                'admin:{}_{}_change'.format(
                    self.model._meta.app_label,
                    self.model._meta.module_name,
                ),
                args=(target.id,),
            )
        )

########NEW FILE########
__FILENAME__ = models
"""
Django Model baseclass for database-backed configuration.
"""
from django.db import models
from django.contrib.auth.models import User
from django.core.cache import get_cache, InvalidCacheBackendError

try:
    cache = get_cache('configuration')  # pylint: disable=invalid-name
except InvalidCacheBackendError:
    from django.core.cache import cache


class ConfigurationModel(models.Model):
    """
    Abstract base class for model-based configuration

    Properties:
        cache_timeout (int): The number of seconds that this configuration
            should be cached
    """

    class Meta(object):  # pylint: disable=missing-docstring
        abstract = True

    # The number of seconds
    cache_timeout = 600

    change_date = models.DateTimeField(auto_now_add=True)
    changed_by = models.ForeignKey(User, editable=False, null=True, on_delete=models.PROTECT)
    enabled = models.BooleanField(default=False)

    def save(self, *args, **kwargs):
        """
        Clear the cached value when saving a new configuration entry
        """
        super(ConfigurationModel, self).save(*args, **kwargs)
        cache.delete(self.cache_key_name())

    @classmethod
    def cache_key_name(cls):
        """Return the name of the key to use to cache the current configuration"""
        return 'configuration/{}/current'.format(cls.__name__)

    @classmethod
    def current(cls):
        """
        Return the active configuration entry, either from cache,
        from the database, or by creating a new empty entry (which is not
        persisted).
        """
        cached = cache.get(cls.cache_key_name())
        if cached is not None:
            return cached

        try:
            current = cls.objects.order_by('-change_date')[0]
        except IndexError:
            current = cls()

        cache.set(cls.cache_key_name(), current, cls.cache_timeout)
        return current

########NEW FILE########
__FILENAME__ = templatetags
"""
Override the submit_row template tag to remove all save buttons from the
admin dashboard change view if the context has readonly marked in it.
"""

from django.contrib.admin.templatetags.admin_modify import register
from django.contrib.admin.templatetags.admin_modify import submit_row as original_submit_row


@register.inclusion_tag('admin/submit_line.html', takes_context=True)
def submit_row(context):
    """
    Overrides 'django.contrib.admin.templatetags.admin_modify.submit_row'.

    Manipulates the context going into that function by hiding all of the buttons
    in the submit row if the key `readonly` is set in the context.
    """
    ctx = original_submit_row(context)

    if context.get('readonly', False):
        ctx.update({
            'show_delete_link': False,
            'show_save_as_new': False,
            'show_save_and_add_another': False,
            'show_save_and_continue': False,
            'show_save': False,
        })
    else:
        return ctx

########NEW FILE########
__FILENAME__ = tests
"""
Tests of ConfigurationModel
"""

from django.contrib.auth.models import User
from django.db import models
from django.test import TestCase

from freezegun import freeze_time

from mock import patch
from config_models.models import ConfigurationModel


class ExampleConfig(ConfigurationModel):
    """
    Test model for testing ``ConfigurationModels``.
    """
    cache_timeout = 300

    string_field = models.TextField()
    int_field = models.IntegerField(default=10)


@patch('config_models.models.cache')
class ConfigurationModelTests(TestCase):
    """
    Tests of ConfigurationModel
    """
    def setUp(self):
        self.user = User()
        self.user.save()

    def test_cache_deleted_on_save(self, mock_cache):
        ExampleConfig(changed_by=self.user).save()
        mock_cache.delete.assert_called_with(ExampleConfig.cache_key_name())

    def test_cache_key_name(self, _mock_cache):
        self.assertEquals(ExampleConfig.cache_key_name(), 'configuration/ExampleConfig/current')

    def test_no_config_empty_cache(self, mock_cache):
        mock_cache.get.return_value = None

        current = ExampleConfig.current()
        self.assertEquals(current.int_field, 10)
        self.assertEquals(current.string_field, '')
        mock_cache.set.assert_called_with(ExampleConfig.cache_key_name(), current, 300)

    def test_no_config_full_cache(self, mock_cache):
        current = ExampleConfig.current()
        self.assertEquals(current, mock_cache.get.return_value)

    def test_config_ordering(self, mock_cache):
        mock_cache.get.return_value = None

        with freeze_time('2012-01-01'):
            first = ExampleConfig(changed_by=self.user)
            first.string_field = 'first'
            first.save()

        second = ExampleConfig(changed_by=self.user)
        second.string_field = 'second'
        second.save()

        self.assertEquals(ExampleConfig.current().string_field, 'second')

    def test_cache_set(self, mock_cache):
        mock_cache.get.return_value = None

        first = ExampleConfig(changed_by=self.user)
        first.string_field = 'first'
        first.save()

        ExampleConfig.current()

        mock_cache.set.assert_called_with(ExampleConfig.cache_key_name(), first, 300)

########NEW FILE########
__FILENAME__ = middleware
from django.http import (HttpResponse, HttpResponseNotModified,
    HttpResponseForbidden)
from student.models import CourseEnrollment

from xmodule.contentstore.django import contentstore
from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
from xmodule.modulestore import InvalidLocationError, InvalidKeyError
from cache_toolbox.core import get_cached_content, set_cached_content
from xmodule.exceptions import NotFoundError

# TODO: Soon as we have a reasonable way to serialize/deserialize AssetKeys, we need
# to change this file so instead of using course_id_partial, we're just using asset keys

class StaticContentServer(object):
    def process_request(self, request):
        # look to see if the request is prefixed with 'c4x' tag
        if request.path.startswith('/' + XASSET_LOCATION_TAG + '/'):
            try:
                loc = StaticContent.get_location_from_path(request.path)
            except (InvalidLocationError, InvalidKeyError):
                # return a 'Bad Request' to browser as we have a malformed Location
                response = HttpResponse()
                response.status_code = 400
                return response

            # first look in our cache so we don't have to round-trip to the DB
            content = get_cached_content(loc)
            if content is None:
                # nope, not in cache, let's fetch from DB
                try:
                    content = contentstore().find(loc, as_stream=True)
                except NotFoundError:
                    response = HttpResponse()
                    response.status_code = 404
                    return response

                # since we fetched it from DB, let's cache it going forward, but only if it's < 1MB
                # this is because I haven't been able to find a means to stream data out of memcached
                if content.length is not None:
                    if content.length < 1048576:
                        # since we've queried as a stream, let's read in the stream into memory to set in cache
                        content = content.copy_to_in_mem()
                        set_cached_content(content)
            else:
                # NOP here, but we may wish to add a "cache-hit" counter in the future
                pass

            # Check that user has access to content
            if getattr(content, "locked", False):
                if not hasattr(request, "user") or not request.user.is_authenticated():
                    return HttpResponseForbidden('Unauthorized')
                if not request.user.is_staff and not CourseEnrollment.is_enrolled_by_partial(
                        request.user, loc.course_key
                ):
                    return HttpResponseForbidden('Unauthorized')

            # convert over the DB persistent last modified timestamp to a HTTP compatible
            # timestamp, so we can simply compare the strings
            last_modified_at_str = content.last_modified_at.strftime("%a, %d-%b-%Y %H:%M:%S GMT")

            # see if the client has cached this content, if so then compare the
            # timestamps, if they are the same then just return a 304 (Not Modified)
            if 'HTTP_IF_MODIFIED_SINCE' in request.META:
                if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
                if if_modified_since == last_modified_at_str:
                    return HttpResponseNotModified()

            response = HttpResponse(content.stream_data(), content_type=content.content_type)
            response['Last-Modified'] = last_modified_at_str

            return response

########NEW FILE########
__FILENAME__ = test
"""
Tests for StaticContentServer
"""
import copy
import logging
from uuid import uuid4
from path import path
from pymongo import MongoClient

from django.contrib.auth.models import User
from django.conf import settings
from django.test.client import Client
from django.test.utils import override_settings

from student.models import CourseEnrollment

from xmodule.contentstore.django import contentstore, _CONTENTSTORE
from xmodule.contentstore.content import StaticContent
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.locations import SlashSeparatedCourseKey
from xmodule.modulestore.tests.django_utils import (studio_store_config,
    ModuleStoreTestCase)
from xmodule.modulestore.xml_importer import import_from_xml

log = logging.getLogger(__name__)

TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex

TEST_MODULESTORE = studio_store_config(settings.TEST_ROOT / "data")


@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE, MODULESTORE=TEST_MODULESTORE)
class ContentStoreToyCourseTest(ModuleStoreTestCase):
    """
    Tests that use the toy course.
    """

    def setUp(self):
        """
        Create user and login.
        """

        settings.MODULESTORE['default']['OPTIONS']['fs_root'] = path('common/test/data')
        settings.MODULESTORE['direct']['OPTIONS']['fs_root'] = path('common/test/data')

        self.client = Client()
        self.contentstore = contentstore()

        self.course_key = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

        import_from_xml(modulestore('direct'), 'common/test/data/', ['toy'],
                static_content_store=self.contentstore, verbose=True)

        # A locked asset
        self.locked_asset = self.course_key.make_asset_key('asset', 'sample_static.txt')
        self.url_locked = self.locked_asset.to_deprecated_string()

        # An unlocked asset
        self.unlocked_asset = self.course_key.make_asset_key('asset', 'another_static.txt')
        self.url_unlocked = self.unlocked_asset.to_deprecated_string()

        self.contentstore.set_attr(self.locked_asset, 'locked', True)

        # Create user
        self.usr = 'testuser'
        self.pwd = 'foo'
        email = 'test+courses@edx.org'
        self.user = User.objects.create_user(self.usr, email, self.pwd)
        self.user.is_active = True
        self.user.save()

        # Create staff user
        self.staff_usr = 'stafftestuser'
        self.staff_pwd = 'foo'
        staff_email = 'stafftest+courses@edx.org'
        self.staff_user = User.objects.create_user(self.staff_usr, staff_email,
                self.staff_pwd)
        self.staff_user.is_active = True
        self.staff_user.is_staff = True
        self.staff_user.save()

    def tearDown(self):

        MongoClient().drop_database(TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'])
        _CONTENTSTORE.clear()

    def test_unlocked_asset(self):
        """
        Test that unlocked assets are being served.
        """
        self.client.logout()
        resp = self.client.get(self.url_unlocked)
        self.assertEqual(resp.status_code, 200) # pylint: disable=E1103

    def test_locked_asset_not_logged_in(self):
        """
        Test that locked assets behave appropriately in case the user is not
        logged in.
        """
        self.client.logout()
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 403) # pylint: disable=E1103

    def test_locked_asset_not_registered(self):
        """
        Test that locked assets behave appropriately in case user is logged in
        in but not registered for the course.
        """
        self.client.login(username=self.usr, password=self.pwd)
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 403) # pylint: disable=E1103

    def test_locked_asset_registered(self):
        """
        Test that locked assets behave appropriately in case user is logged in
        and registered for the course.
        """
        CourseEnrollment.enroll(self.user, self.course_key)
        self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key))

        self.client.login(username=self.usr, password=self.pwd)
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 200) # pylint: disable=E1103

    def test_locked_asset_staff(self):
        """
        Test that locked assets behave appropriately in case user is staff.
        """
        self.client.login(username=self.staff_usr, password=self.staff_pwd)
        resp = self.client.get(self.url_locked)
        self.assertEqual(resp.status_code, 200) # pylint: disable=E1103


########NEW FILE########
__FILENAME__ = cohorts
"""
This file contains the logic for cohort groups, as exposed internally to the
forums, and to the cohort admin views.
"""

from django.http import Http404
import logging
import random

from courseware import courses
from student.models import get_user_by_username_or_email
from .models import CourseUserGroup

log = logging.getLogger(__name__)


# tl;dr: global state is bad.  capa reseeds random every time a problem is loaded.  Even
# if and when that's fixed, it's a good idea to have a local generator to avoid any other
# code that messes with the global random module.
_local_random = None

def local_random():
    """
    Get the local random number generator.  In a function so that we don't run
    random.Random() at import time.
    """
    # ironic, isn't it?
    global _local_random

    if _local_random is None:
        _local_random = random.Random()

    return _local_random


def is_course_cohorted(course_key):
    """
    Given a course key, return a boolean for whether or not the course is
    cohorted.

    Raises:
       Http404 if the course doesn't exist.
    """
    return courses.get_course_by_id(course_key).is_cohorted


def get_cohort_id(user, course_key):
    """
    Given a course key and a user, return the id of the cohort that user is
    assigned to in that course.  If they don't have a cohort, return None.
    """
    cohort = get_cohort(user, course_key)
    return None if cohort is None else cohort.id


def is_commentable_cohorted(course_key, commentable_id):
    """
    Args:
        course_key: CourseKey
        commentable_id: string

    Returns:
        Bool: is this commentable cohorted?

    Raises:
        Http404 if the course doesn't exist.
    """
    course = courses.get_course_by_id(course_key)

    if not course.is_cohorted:
        # this is the easy case :)
        ans = False
    elif commentable_id in course.top_level_discussion_topic_ids:
        # top level discussions have to be manually configured as cohorted
        # (default is not)
        ans = commentable_id in course.cohorted_discussions
    else:
        # inline discussions are cohorted by default
        ans = True

    log.debug(u"is_commentable_cohorted({0}, {1}) = {2}".format(
        course_key, commentable_id, ans
    ))
    return ans


def get_cohorted_commentables(course_key):
    """
    Given a course_key return a list of strings representing cohorted commentables
    """

    course = courses.get_course_by_id(course_key)

    if not course.is_cohorted:
        # this is the easy case :)
        ans = []
    else:
        ans = course.cohorted_discussions

    return ans


def get_cohort(user, course_key):
    """
    Given a django User and a CourseKey, return the user's cohort in that
    cohort.

    Arguments:
        user: a Django User object.
        course_key: CourseKey

    Returns:
        A CourseUserGroup object if the course is cohorted and the User has a
        cohort, else None.

    Raises:
       ValueError if the CourseKey doesn't exist.
    """
    # First check whether the course is cohorted (users shouldn't be in a cohort
    # in non-cohorted courses, but settings can change after course starts)
    try:
        course = courses.get_course_by_id(course_key)
    except Http404:
        raise ValueError("Invalid course_key")

    if not course.is_cohorted:
        return None

    try:
        return CourseUserGroup.objects.get(course_id=course_key,
                                            group_type=CourseUserGroup.COHORT,
                                            users__id=user.id)
    except CourseUserGroup.DoesNotExist:
        # Didn't find the group.  We'll go on to create one if needed.
        pass

    if not course.auto_cohort:
        return None

    choices = course.auto_cohort_groups
    n = len(choices)
    if n == 0:
        # Nowhere to put user
        log.warning("Course %s is auto-cohorted, but there are no"
                    " auto_cohort_groups specified",
                    course_key)
        return None

    # Put user in a random group, creating it if needed
    group_name = local_random().choice(choices)

    group, created = CourseUserGroup.objects.get_or_create(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        name=group_name
    )

    user.course_groups.add(group)
    return group


def get_course_cohorts(course_key):
    """
    Get a list of all the cohorts in the given course.

    Arguments:
        course_key: CourseKey

    Returns:
        A list of CourseUserGroup objects.  Empty if there are no cohorts. Does
        not check whether the course is cohorted.
    """
    return list(CourseUserGroup.objects.filter(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT
    ))

### Helpers for cohort management views


def get_cohort_by_name(course_key, name):
    """
    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
    it isn't present.
    """
    return CourseUserGroup.objects.get(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        name=name
    )


def get_cohort_by_id(course_key, cohort_id):
    """
    Return the CourseUserGroup object for the given cohort.  Raises DoesNotExist
    it isn't present.  Uses the course_key for extra validation...
    """
    return CourseUserGroup.objects.get(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        id=cohort_id
    )


def add_cohort(course_key, name):
    """
    Add a cohort to a course.  Raises ValueError if a cohort of the same name already
    exists.
    """
    log.debug("Adding cohort %s to %s", name, course_key)
    if CourseUserGroup.objects.filter(course_id=course_key,
                                      group_type=CourseUserGroup.COHORT,
                                      name=name).exists():
        raise ValueError("Can't create two cohorts with the same name")

    return CourseUserGroup.objects.create(
        course_id=course_key,
        group_type=CourseUserGroup.COHORT,
        name=name
    )


class CohortConflict(Exception):
    """
    Raised when user to be added is already in another cohort in same course.
    """
    pass


def add_user_to_cohort(cohort, username_or_email):
    """
    Look up the given user, and if successful, add them to the specified cohort.

    Arguments:
        cohort: CourseUserGroup
        username_or_email: string.  Treated as email if has '@'

    Returns:
        Tuple of User object and string (or None) indicating previous cohort

    Raises:
        User.DoesNotExist if can't find user.

        ValueError if user already present in this cohort.
    """
    user = get_user_by_username_or_email(username_or_email)
    previous_cohort = None

    course_cohorts = CourseUserGroup.objects.filter(
        course_id=cohort.course_id,
        users__id=user.id,
        group_type=CourseUserGroup.COHORT
    )
    if course_cohorts.exists():
        if course_cohorts[0] == cohort:
            raise ValueError("User {0} already present in cohort {1}".format(
                user.username,
                cohort.name))
        else:
            previous_cohort = course_cohorts[0].name
            course_cohorts[0].users.remove(user)

    cohort.users.add(user)
    return (user, previous_cohort)


def get_course_cohort_names(course_key):
    """
    Return a list of the cohort names in a course.
    """
    return [c.name for c in get_course_cohorts(course_key)]


def delete_empty_cohort(course_key, name):
    """
    Remove an empty cohort.  Raise ValueError if cohort is not empty.
    """
    cohort = get_cohort_by_name(course_key, name)
    if cohort.users.exists():
        raise ValueError(
            "Can't delete non-empty cohort {0} in course {1}".format(
                name, course_key))

    cohort.delete()

########NEW FILE########
__FILENAME__ = models
import logging

from django.contrib.auth.models import User
from django.db import models
from xmodule_django.models import CourseKeyField

log = logging.getLogger(__name__)


class CourseUserGroup(models.Model):
    """
    This model represents groups of users in a course.  Groups may have different types,
    which may be treated specially.  For example, a user can be in at most one cohort per
    course, and cohorts are used to split up the forums by group.
    """
    class Meta:
        unique_together = (('name', 'course_id'), )

    name = models.CharField(max_length=255,
                            help_text=("What is the name of this group?  "
                                       "Must be unique within a course."))
    users = models.ManyToManyField(User, db_index=True, related_name='course_groups',
                                   help_text="Who is in this group?")

    # Note: groups associated with particular runs of a course.  E.g. Fall 2012 and Spring
    # 2013 versions of 6.00x will have separate groups.
    course_id = CourseKeyField(max_length=255, db_index=True,
                                 help_text="Which course is this group associated with?")

    # For now, only have group type 'cohort', but adding a type field to support
    # things like 'question_discussion', 'friends', 'off-line-class', etc
    COHORT = 'cohort'
    GROUP_TYPE_CHOICES = ((COHORT, 'Cohort'),)
    group_type = models.CharField(max_length=20, choices=GROUP_TYPE_CHOICES)

########NEW FILE########
__FILENAME__ = test_cohorts
import django.test
from django.contrib.auth.models import User
from django.conf import settings

from django.test.utils import override_settings

from course_groups.models import CourseUserGroup
from course_groups.cohorts import (get_cohort, get_course_cohorts,
                                   is_commentable_cohorted, get_cohort_by_name)

from xmodule.modulestore.django import modulestore, clear_existing_modulestores
from xmodule.modulestore.locations import SlashSeparatedCourseKey

from xmodule.modulestore.tests.django_utils import mixed_store_config

# NOTE: running this with the lms.envs.test config works without
# manually overriding the modulestore.  However, running with
# cms.envs.test doesn't.

TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
TEST_MAPPING = {'edX/toy/2012_Fall': 'xml'}
TEST_DATA_MIXED_MODULESTORE = mixed_store_config(TEST_DATA_DIR, TEST_MAPPING)


@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class TestCohorts(django.test.TestCase):

    @staticmethod
    def topic_name_to_id(course, name):
        """
        Given a discussion topic name, return an id for that name (includes
        course and url_name).
        """
        return "{course}_{run}_{name}".format(course=course.location.course,
                                              run=course.url_name,
                                              name=name)

    @staticmethod
    def config_course_cohorts(course, discussions,
                              cohorted,
                              cohorted_discussions=None,
                              auto_cohort=None,
                              auto_cohort_groups=None):
        """
        Given a course with no discussion set up, add the discussions and set
        the cohort config appropriately.

        Arguments:
            course: CourseDescriptor
            discussions: list of topic names strings.  Picks ids and sort_keys
                automatically.
            cohorted: bool.
            cohorted_discussions: optional list of topic names.  If specified,
                converts them to use the same ids as topic names.
            auto_cohort: optional bool.
            auto_cohort_groups: optional list of strings
                      (names of groups to put students into).

        Returns:
            Nothing -- modifies course in place.
        """
        def to_id(name):
            return TestCohorts.topic_name_to_id(course, name)

        topics = dict((name, {"sort_key": "A",
                              "id": to_id(name)})
                      for name in discussions)

        course.discussion_topics = topics

        d = {"cohorted": cohorted}
        if cohorted_discussions is not None:
            d["cohorted_discussions"] = [to_id(name)
                                         for name in cohorted_discussions]

        if auto_cohort is not None:
            d["auto_cohort"] = auto_cohort
        if auto_cohort_groups is not None:
            d["auto_cohort_groups"] = auto_cohort_groups

        course.cohort_config = d

    def setUp(self):
        """
        Make sure that course is reloaded every time--clear out the modulestore.
        """
        clear_existing_modulestores()
        self.toy_course_key = SlashSeparatedCourseKey("edX", "toy", "2012_Fall")

    def test_get_cohort(self):
        """
        Make sure get_cohort() does the right thing when the course is cohorted
        """
        course = modulestore().get_course(self.toy_course_key)
        self.assertEqual(course.id, self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        user = User.objects.create(username="test", email="a@b.com")
        other_user = User.objects.create(username="test2", email="a2@b.com")

        self.assertIsNone(get_cohort(user, course.id), "No cohort created yet")

        cohort = CourseUserGroup.objects.create(name="TestCohort",
                                                course_id=course.id,
                                                group_type=CourseUserGroup.COHORT)

        cohort.users.add(user)

        self.assertIsNone(get_cohort(user, course.id),
                          "Course isn't cohorted, so shouldn't have a cohort")

        # Make the course cohorted...
        self.config_course_cohorts(course, [], cohorted=True)

        self.assertEquals(get_cohort(user, course.id).id, cohort.id,
                          "Should find the right cohort")

        self.assertEquals(get_cohort(other_user, course.id), None,
                          "other_user shouldn't have a cohort")

    def test_auto_cohorting(self):
        """
        Make sure get_cohort() does the right thing when the course is auto_cohorted
        """
        course = modulestore().get_course(self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        user1 = User.objects.create(username="test", email="a@b.com")
        user2 = User.objects.create(username="test2", email="a2@b.com")
        user3 = User.objects.create(username="test3", email="a3@b.com")

        cohort = CourseUserGroup.objects.create(name="TestCohort",
                                                course_id=course.id,
                                                group_type=CourseUserGroup.COHORT)

        # user1 manually added to a cohort
        cohort.users.add(user1)

        # Make the course auto cohorted...
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=["AutoGroup"])

        self.assertEquals(get_cohort(user1, course.id).id, cohort.id,
                          "user1 should stay put")

        self.assertEquals(get_cohort(user2, course.id).name, "AutoGroup",
                          "user2 should be auto-cohorted")

        # Now make the group list empty
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=[])

        self.assertEquals(get_cohort(user3, course.id), None,
                          "No groups->no auto-cohorting")

        # Now make it different
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=["OtherGroup"])

        self.assertEquals(get_cohort(user3, course.id).name, "OtherGroup",
                          "New list->new group")
        self.assertEquals(get_cohort(user2, course.id).name, "AutoGroup",
                          "user2 should still be in originally placed cohort")

    def test_auto_cohorting_randomization(self):
        """
        Make sure get_cohort() randomizes properly.
        """
        course = modulestore().get_course(self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        groups = ["group_{0}".format(n) for n in range(5)]
        self.config_course_cohorts(course, [], cohorted=True,
                                   auto_cohort=True,
                                   auto_cohort_groups=groups)

        # Assign 100 users to cohorts
        for i in range(100):
            user = User.objects.create(username="test_{0}".format(i),
                                       email="a@b{0}.com".format(i))
            get_cohort(user, course.id)

        # Now make sure that the assignment was at least vaguely random:
        # each cohort should have at least 1, and fewer than 50 students.
        # (with 5 groups, probability of 0 users in any group is about
        # .8**100= 2.0e-10)
        for cohort_name in groups:
            cohort = get_cohort_by_name(course.id, cohort_name)
            num_users = cohort.users.count()
            self.assertGreater(num_users, 1)
            self.assertLess(num_users, 50)

    def test_get_course_cohorts(self):
        course1_key = SlashSeparatedCourseKey('a', 'b', 'c')
        course2_key = SlashSeparatedCourseKey('e', 'f', 'g')

        # add some cohorts to course 1
        cohort = CourseUserGroup.objects.create(name="TestCohort",
                                                course_id=course1_key,
                                                group_type=CourseUserGroup.COHORT)

        cohort = CourseUserGroup.objects.create(name="TestCohort2",
                                                course_id=course1_key,
                                                group_type=CourseUserGroup.COHORT)

        # second course should have no cohorts
        self.assertEqual(get_course_cohorts(course2_key), [])

        cohorts = sorted([c.name for c in get_course_cohorts(course1_key)])
        self.assertEqual(cohorts, ['TestCohort', 'TestCohort2'])

    def test_is_commentable_cohorted(self):
        course = modulestore().get_course(self.toy_course_key)
        self.assertFalse(course.is_cohorted)

        def to_id(name):
            return self.topic_name_to_id(course, name)

        # no topics
        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course doesn't even have a 'General' topic")

        # not cohorted
        self.config_course_cohorts(course, ["General", "Feedback"],
                                   cohorted=False)

        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course isn't cohorted")

        # cohorted, but top level topics aren't
        self.config_course_cohorts(course, ["General", "Feedback"],
                                   cohorted=True)

        self.assertTrue(course.is_cohorted)
        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course is cohorted, but 'General' isn't.")

        self.assertTrue(
            is_commentable_cohorted(course.id, to_id("random")),
            "Non-top-level discussion is always cohorted in cohorted courses.")

        # cohorted, including "Feedback" top-level topics aren't
        self.config_course_cohorts(course, ["General", "Feedback"],
                                   cohorted=True,
                                   cohorted_discussions=["Feedback"])

        self.assertTrue(course.is_cohorted)
        self.assertFalse(is_commentable_cohorted(course.id, to_id("General")),
                         "Course is cohorted, but 'General' isn't.")

        self.assertTrue(
            is_commentable_cohorted(course.id, to_id("Feedback")),
            "Feedback was listed as cohorted.  Should be.")

########NEW FILE########
__FILENAME__ = test_views
import json

from django.test.client import RequestFactory
from django.test.utils import override_settings
from factory import post_generation, Sequence
from factory.django import DjangoModelFactory

from course_groups.models import CourseUserGroup
from course_groups.views import add_users_to_cohort
from courseware.tests.tests import TEST_DATA_MIXED_MODULESTORE
from student.tests.factories import UserFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory


class CohortFactory(DjangoModelFactory):
    FACTORY_FOR = CourseUserGroup

    name = Sequence("cohort{}".format)
    course_id = "dummy_id"
    group_type = CourseUserGroup.COHORT

    @post_generation
    def users(self, create, extracted, **kwargs):  # pylint: disable=W0613
        self.users.add(*extracted)


@override_settings(MODULESTORE=TEST_DATA_MIXED_MODULESTORE)
class AddUsersToCohortTestCase(ModuleStoreTestCase):
    def setUp(self):
        self.course = CourseFactory.create()
        self.staff_user = UserFactory.create(is_staff=True)
        self.cohort1_users = [UserFactory.create() for _ in range(3)]
        self.cohort2_users = [UserFactory.create() for _ in range(2)]
        self.cohort3_users = [UserFactory.create() for _ in range(2)]
        self.cohortless_users = [UserFactory.create() for _ in range(3)]
        self.cohort1 = CohortFactory.create(course_id=self.course.id, users=self.cohort1_users)
        self.cohort2 = CohortFactory.create(course_id=self.course.id, users=self.cohort2_users)
        self.cohort3 = CohortFactory.create(course_id=self.course.id, users=self.cohort3_users)

    def check_request(
            self,
            users_string,
            expected_added=None,
            expected_changed=None,
            expected_present=None,
            expected_unknown=None
    ):
        """
        Check that add_users_to_cohort returns the expected result and has the
        expected side effects. The given users will be added to cohort1.

        users_string is the string input entered by the client

        expected_added is a list of users

        expected_changed is a list of (user, previous_cohort) tuples

        expected_present is a list of (user, email/username) tuples where
        email/username corresponds to the input

        expected_unknown is a list of strings corresponding to the input
        """
        expected_added = expected_added or []
        expected_changed = expected_changed or []
        expected_present = expected_present or []
        expected_unknown = expected_unknown or []
        request = RequestFactory().post("dummy_url", {"users": users_string})
        request.user = self.staff_user
        response = add_users_to_cohort(request, self.course.id.to_deprecated_string(), self.cohort1.id)
        self.assertEqual(response.status_code, 200)
        result = json.loads(response.content)
        self.assertEqual(result.get("success"), True)
        self.assertItemsEqual(
            result.get("added"),
            [
                {"username": user.username, "name": user.profile.name, "email": user.email}
                for user in expected_added
            ]
        )
        self.assertItemsEqual(
            result.get("changed"),
            [
                {
                    "username": user.username,
                    "name": user.profile.name,
                    "email": user.email,
                    "previous_cohort": previous_cohort
                }
                for (user, previous_cohort) in expected_changed
            ]
        )
        self.assertItemsEqual(
            result.get("present"),
            [username_or_email for (_, username_or_email) in expected_present]
        )
        self.assertItemsEqual(result.get("unknown"), expected_unknown)
        for user in expected_added + [user for (user, _) in expected_changed + expected_present]:
            self.assertEqual(
                CourseUserGroup.objects.get(
                    course_id=self.course.id,
                    group_type=CourseUserGroup.COHORT,
                    users__id=user.id
                ),
                self.cohort1
            )

    def test_empty(self):
        self.check_request("")

    def test_only_added(self):
        self.check_request(
            ",".join([user.username for user in self.cohortless_users]),
            expected_added=self.cohortless_users
        )

    def test_only_changed(self):
        self.check_request(
            ",".join([user.username for user in self.cohort2_users + self.cohort3_users]),
            expected_changed=(
                [(user, self.cohort2.name) for user in self.cohort2_users] +
                [(user, self.cohort3.name) for user in self.cohort3_users]
            )
        )

    def test_only_present(self):
        usernames = [user.username for user in self.cohort1_users]
        self.check_request(
            ",".join(usernames),
            expected_present=[(user, user.username) for user in self.cohort1_users]
        )

    def test_only_unknown(self):
        usernames = ["unknown_user{}".format(i) for i in range(3)]
        self.check_request(
            ",".join(usernames),
            expected_unknown=usernames
        )

    def test_all(self):
        unknowns = ["unknown_user{}".format(i) for i in range(3)]
        self.check_request(
            ",".join(
                unknowns +
                [
                    user.username
                    for user in self.cohortless_users + self.cohort1_users + self.cohort2_users + self.cohort3_users
                ]
            ),
            self.cohortless_users,
            (
                [(user, self.cohort2.name) for user in self.cohort2_users] +
                [(user, self.cohort3.name) for user in self.cohort3_users]
            ),
            [(user, user.username) for user in self.cohort1_users],
            unknowns
        )

    def test_emails(self):
        unknown = "unknown_user@example.com"
        self.check_request(
            ",".join([
                self.cohort1_users[0].email,
                self.cohort2_users[0].email,
                self.cohortless_users[0].email,
                unknown
            ]),
            [self.cohortless_users[0]],
            [(self.cohort2_users[0], self.cohort2.name)],
            [(self.cohort1_users[0], self.cohort1_users[0].email)],
            [unknown]
        )

    def test_delimiters(self):
        unknown = "unknown_user"
        self.check_request(
            " {} {}\t{}, \r\n{}".format(
                unknown,
                self.cohort1_users[0].username,
                self.cohort2_users[0].username,
                self.cohortless_users[0].username
            ),
            [self.cohortless_users[0]],
            [(self.cohort2_users[0], self.cohort2.name)],
            [(self.cohort1_users[0], self.cohort1_users[0].username)],
            [unknown]
        )

########NEW FILE########
__FILENAME__ = views
from django_future.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_POST
from django.contrib.auth.models import User
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.core.urlresolvers import reverse
from django.http import HttpResponse
import json
import logging
import re

from xmodule.modulestore.locations import SlashSeparatedCourseKey
from courseware.courses import get_course_with_access
from edxmako.shortcuts import render_to_response

from . import cohorts


log = logging.getLogger(__name__)


def json_http_response(data):
    """
    Return an HttpResponse with the data json-serialized and the right content
    type header.
    """
    return HttpResponse(json.dumps(data), content_type="application/json")


def split_by_comma_and_whitespace(s):
    """
    Split a string both by commas and whitespice.  Returns a list.
    """
    return re.split(r'[\s,]+', s)


@ensure_csrf_cookie
def list_cohorts(request, course_key):
    """
    Return json dump of dict:

    {'success': True,
     'cohorts': [{'name': name, 'id': id}, ...]}
    """

    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)

    get_course_with_access(request.user, 'staff', course_key)

    all_cohorts = [{'name': c.name, 'id': c.id}
                   for c in cohorts.get_course_cohorts(course_key)]

    return json_http_response({'success': True,
                               'cohorts': all_cohorts})


@ensure_csrf_cookie
@require_POST
def add_cohort(request, course_key):
    """
    Return json of dict:
    {'success': True,
     'cohort': {'id': id,
                'name': name}}

                or

    {'success': False,
     'msg': error_msg} if there's an error
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)

    get_course_with_access(request.user, 'staff', course_key)

    name = request.POST.get("name")
    if not name:
        return json_http_response({'success': False,
                                'msg': "No name specified"})

    try:
        cohort = cohorts.add_cohort(course_key, name)
    except ValueError as err:
        return json_http_response({'success': False,
                                'msg': str(err)})

    return json_http_response({'success': 'True',
                            'cohort': {
                                'id': cohort.id,
                                'name': cohort.name
                                }})


@ensure_csrf_cookie
def users_in_cohort(request, course_key, cohort_id):
    """
    Return users in the cohort.  Show up to 100 per page, and page
    using the 'page' GET attribute in the call.  Format:

    Returns:
        Json dump of dictionary in the following format:
        {'success': True,
         'page': page,
         'num_pages': paginator.num_pages,
         'users': [{'username': ..., 'email': ..., 'name': ...}]
    }
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)

    get_course_with_access(request.user, 'staff', course_key)

    # this will error if called with a non-int cohort_id.  That's ok--it
    # shoudn't happen for valid clients.
    cohort = cohorts.get_cohort_by_id(course_key, int(cohort_id))

    paginator = Paginator(cohort.users.all(), 100)
    page = request.GET.get('page')
    try:
        users = paginator.page(page)
    except PageNotAnInteger:
        # return the first page
        page = 1
        users = paginator.page(page)
    except EmptyPage:
        # Page is out of range.  Return last page
        page = paginator.num_pages
        contacts = paginator.page(page)

    user_info = [{'username': u.username,
                  'email': u.email,
                  'name': '{0} {1}'.format(u.first_name, u.last_name)}
                 for u in users]

    return json_http_response({'success': True,
                               'page': page,
                               'num_pages': paginator.num_pages,
                               'users': user_info})


@ensure_csrf_cookie
@require_POST
def add_users_to_cohort(request, course_key, cohort_id):
    """
    Return json dict of:

    {'success': True,
     'added': [{'username': ...,
                'name': ...,
                'email': ...}, ...],
     'changed': [{'username': ...,
                  'name': ...,
                  'email': ...,
                  'previous_cohort': ...}, ...],
     'present': [str1, str2, ...],    # already there
     'unknown': [str1, str2, ...]}
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)
    get_course_with_access(request.user, 'staff', course_key)

    cohort = cohorts.get_cohort_by_id(course_key, cohort_id)

    users = request.POST.get('users', '')
    added = []
    changed = []
    present = []
    unknown = []
    for username_or_email in split_by_comma_and_whitespace(users):
        if not username_or_email:
            continue

        try:
            (user, previous_cohort) = cohorts.add_user_to_cohort(cohort, username_or_email)
            info = {
                'username': user.username,
                'name': user.profile.name,
                'email': user.email,
            }
            if previous_cohort:
                info['previous_cohort'] = previous_cohort
                changed.append(info)
            else:
                added.append(info)
        except ValueError:
            present.append(username_or_email)
        except User.DoesNotExist:
            unknown.append(username_or_email)

    return json_http_response({'success': True,
                               'added': added,
                               'changed': changed,
                               'present': present,
                               'unknown': unknown})


@ensure_csrf_cookie
@require_POST
def remove_user_from_cohort(request, course_key, cohort_id):
    """
    Expects 'username': username in POST data.

    Return json dict of:

    {'success': True} or
    {'success': False,
     'msg': error_msg}
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)
    get_course_with_access(request.user, 'staff', course_key)

    username = request.POST.get('username')
    if username is None:
        return json_http_response({'success': False,
                                   'msg': 'No username specified'})

    cohort = cohorts.get_cohort_by_id(course_key, cohort_id)
    try:
        user = User.objects.get(username=username)
        cohort.users.remove(user)
        return json_http_response({'success': True})
    except User.DoesNotExist:
        log.debug('no user')
        return json_http_response({'success': False,
                                   'msg': "No user '{0}'".format(username)})


def debug_cohort_mgmt(request, course_key):
    """
    Debugging view for dev.
    """
    # this is a string when we get it here
    course_key = SlashSeparatedCourseKey.from_deprecated_string(course_key)
    # add staff check to make sure it's safe if it's accidentally deployed.
    get_course_with_access(request.user, 'staff', course_key)

    context = {'cohorts_ajax_url': reverse(
        'cohorts',
        kwargs={'course_key': course_key.to_deprecated_string()}
    )}
    return render_to_response('/course_groups/debug.html', context)

########NEW FILE########
__FILENAME__ = admin
from ratelimitbackend import admin
from course_modes.models import CourseMode
from django import forms

from opaque_keys import InvalidKeyError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.keys import CourseKey
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class CourseModeForm(forms.ModelForm):

    class Meta:
        model = CourseMode

    def clean_course_id(self):
        course_id = self.cleaned_data['course_id']
        try:
            course_key = CourseKey.from_string(course_id)
        except InvalidKeyError:
            try:
                course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
            except InvalidKeyError:
                raise forms.ValidationError("Cannot make a valid CourseKey from id {}!".format(course_id))

        if not modulestore().has_course(course_key):
            raise forms.ValidationError("Cannot find course with id {} in the modulestore".format(course_id))

        return course_key


class CourseModeAdmin(admin.ModelAdmin):
    form = CourseModeForm


admin.site.register(CourseMode, CourseModeAdmin)

########NEW FILE########
__FILENAME__ = 0001_initial
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'CourseMode'
        db.create_table('course_modes_coursemode', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('course_id', self.gf('django.db.models.fields.CharField')(max_length=255, db_index=True)),
            ('mode_slug', self.gf('django.db.models.fields.CharField')(max_length=100)),
            ('mode_display_name', self.gf('django.db.models.fields.CharField')(max_length=255)),
            ('min_price', self.gf('django.db.models.fields.IntegerField')(default=0)),
            ('suggested_prices', self.gf('django.db.models.fields.CommaSeparatedIntegerField')(default='', max_length=255, blank=True)),
        ))
        db.send_create_signal('course_modes', ['CourseMode'])


    def backwards(self, orm):
        # Deleting model 'CourseMode'
        db.delete_table('course_modes_coursemode')


    models = {
        'course_modes.coursemode': {
            'Meta': {'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0002_auto__add_field_coursemode_currency
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'CourseMode.currency'
        db.add_column('course_modes_coursemode', 'currency',
                      self.gf('django.db.models.fields.CharField')(default='usd', max_length=8),
                      keep_default=False)


    def backwards(self, orm):
        # Deleting field 'CourseMode.currency'
        db.delete_column('course_modes_coursemode', 'currency')


    models = {
        'course_modes.coursemode': {
            'Meta': {'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0003_auto__add_unique_coursemode_course_id_currency_mode_slug
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding unique constraint on 'CourseMode', fields ['course_id', 'currency', 'mode_slug']
        db.create_unique('course_modes_coursemode', ['course_id', 'currency', 'mode_slug'])


    def backwards(self, orm):
        # Removing unique constraint on 'CourseMode', fields ['course_id', 'currency', 'mode_slug']
        db.delete_unique('course_modes_coursemode', ['course_id', 'currency', 'mode_slug'])


    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0004_auto__add_field_coursemode_expiration_date
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'CourseMode.expiration_date'
        db.add_column('course_modes_coursemode', 'expiration_date',
                      self.gf('django.db.models.fields.DateField')(default=None, null=True),
                      keep_default=False)


    def backwards(self, orm):
        # Deleting field 'CourseMode.expiration_date'
        db.delete_column('course_modes_coursemode', 'expiration_date')


    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'expiration_date': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0005_auto__add_field_coursemode_expiration_datetime
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding field 'CourseMode.expiration_datetime'
        db.add_column('course_modes_coursemode', 'expiration_datetime',
                      self.gf('django.db.models.fields.DateTimeField')(default=None, null=True, blank=True),
                      keep_default=False)


    def backwards(self, orm):
        # Deleting field 'CourseMode.expiration_datetime'
        db.delete_column('course_modes_coursemode', 'expiration_datetime')


    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'expiration_date': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'expiration_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
########NEW FILE########
__FILENAME__ = 0006_expiration_date_to_datetime
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        from datetime import datetime
        for course_mode in orm.CourseMode.objects.all():
            if course_mode.expiration_date is None:
                course_mode.expiration_datetime = None
                course_mode.save()
            else:
                course_mode.expiration_datetime = datetime.combine(course_mode.expiration_date, datetime.min.time())
                course_mode.save()

    def backwards(self, orm):
        for course_mode in orm.CourseMode.objects.all():
            course_mode.expiration_datetime = None
            course_mode.save()

    models = {
        'course_modes.coursemode': {
            'Meta': {'unique_together': "(('course_id', 'mode_slug', 'currency'),)", 'object_name': 'CourseMode'},
            'course_id': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
            'currency': ('django.db.models.fields.CharField', [], {'default': "'usd'", 'max_length': '8'}),
            'expiration_date': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'expiration_datetime': ('django.db.models.fields.DateTimeField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'min_price': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
            'mode_display_name': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
            'mode_slug': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'suggested_prices': ('django.db.models.fields.CommaSeparatedIntegerField', [], {'default': "''", 'max_length': '255', 'blank': 'True'})
        }
    }

    complete_apps = ['course_modes']
    symmetrical = True

########NEW FILE########
__FILENAME__ = models
"""
Add and create new modes for running courses on this particular LMS
"""
import pytz
from datetime import datetime

from django.db import models
from collections import namedtuple
from django.utils.translation import ugettext as _
from django.db.models import Q

from xmodule_django.models import CourseKeyField

Mode = namedtuple('Mode', ['slug', 'name', 'min_price', 'suggested_prices', 'currency', 'expiration_datetime'])

class CourseMode(models.Model):
    """
    We would like to offer a course in a variety of modes.

    """
    # the course that this mode is attached to
    course_id = CourseKeyField(max_length=255, db_index=True)

    # the reference to this mode that can be used by Enrollments to generate
    # similar behavior for the same slug across courses
    mode_slug = models.CharField(max_length=100)

    # The 'pretty' name that can be translated and displayed
    mode_display_name = models.CharField(max_length=255)

    # minimum price in USD that we would like to charge for this mode of the course
    min_price = models.IntegerField(default=0)

    # the suggested prices for this mode
    suggested_prices = models.CommaSeparatedIntegerField(max_length=255, blank=True, default='')

    # the currency these prices are in, using lower case ISO currency codes
    currency = models.CharField(default="usd", max_length=8)

    # turn this mode off after the given expiration date
    expiration_date = models.DateField(default=None, null=True, blank=True)

    expiration_datetime = models.DateTimeField(default=None, null=True, blank=True)

    DEFAULT_MODE = Mode('honor', _('Honor Code Certificate'), 0, '', 'usd', None)
    DEFAULT_MODE_SLUG = 'honor'

    class Meta:
        """ meta attributes of this model """
        unique_together = ('course_id', 'mode_slug', 'currency')

    @classmethod
    def modes_for_course(cls, course_id):
        """
        Returns a list of the non-expired modes for a given course id

        If no modes have been set in the table, returns the default mode
        """
        now = datetime.now(pytz.UTC)
        found_course_modes = cls.objects.filter(Q(course_id=course_id) &
                                                (Q(expiration_datetime__isnull=True) |
                                                Q(expiration_datetime__gte=now)))
        modes = ([Mode(
            mode.mode_slug,
            mode.mode_display_name,
            mode.min_price,
            mode.suggested_prices,
            mode.currency,
            mode.expiration_datetime
        ) for mode in found_course_modes])
        if not modes:
            modes = [cls.DEFAULT_MODE]
        return modes

    @classmethod
    def modes_for_course_dict(cls, course_id):
        """
        Returns the non-expired modes for a particular course as a
        dictionary with the mode slug as the key
        """
        return {mode.slug: mode for mode in cls.modes_for_course(course_id)}

    @classmethod
    def mode_for_course(cls, course_id, mode_slug):
        """
        Returns the mode for the course corresponding to mode_slug.

        Returns only non-expired modes.

        If this particular mode is not set for the course, returns None
        """
        modes = cls.modes_for_course(course_id)

        matched = [m for m in modes if m.slug == mode_slug]
        if matched:
            return matched[0]
        else:
            return None

    @classmethod
    def min_course_price_for_verified_for_currency(cls, course_id, currency):
        """
        Returns the minimum price of the course int he appropriate currency over all the
        course's *verified*, non-expired modes.

        Assuming all verified courses have a minimum price of >0, this value should always
        be >0.

        If no verified mode is found, 0 is returned.
        """
        modes = cls.modes_for_course(course_id)
        for mode in modes:
            if (mode.currency == currency) and (mode.slug == 'verified'):
                return mode.min_price
        return 0

    @classmethod
    def min_course_price_for_currency(cls, course_id, currency):
        """
        Returns the minimum price of the course in the appropriate currency over all the course's
        non-expired modes.
        If there is no mode found, will return the price of DEFAULT_MODE, which is 0
        """
        modes = cls.modes_for_course(course_id)
        return min(mode.min_price for mode in modes if mode.currency == currency)

    def __unicode__(self):
        return u"{} : {}, min={}, prices={}".format(
            self.course_id.to_deprecated_string(), self.mode_slug, self.min_price, self.suggested_prices
        )

########NEW FILE########
__FILENAME__ = factories
from course_modes.models import CourseMode
from factory.django import DjangoModelFactory


# Factories don't have __init__ methods, and are self documenting
# pylint: disable=W0232
class CourseModeFactory(DjangoModelFactory):
    FACTORY_FOR = CourseMode

    course_id = u'MITx/999/Robot_Super_Course'
    mode_slug = 'audit'
    mode_display_name = 'audit course'
    min_price = 0
    currency = 'usd'
    expiration_datetime = None

########NEW FILE########
__FILENAME__ = test_models
"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from datetime import datetime, timedelta
import pytz

from xmodule.modulestore.locations import SlashSeparatedCourseKey
from django.test import TestCase
from course_modes.models import CourseMode, Mode


class CourseModeModelTest(TestCase):
    """
    Tests for the CourseMode model
    """

    def setUp(self):
        self.course_key = SlashSeparatedCourseKey('Test', 'TestCourse', 'TestCourseRun')
        CourseMode.objects.all().delete()

    def create_mode(self, mode_slug, mode_name, min_price=0, suggested_prices='', currency='usd'):
        """
        Create a new course mode
        """
        return CourseMode.objects.get_or_create(
            course_id=self.course_key,
            mode_display_name=mode_name,
            mode_slug=mode_slug,
            min_price=min_price,
            suggested_prices=suggested_prices,
            currency=currency,
        )

    def test_modes_for_course_empty(self):
        """
        If we can't find any modes, we should get back the default mode
        """
        # shouldn't be able to find a corresponding course
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([CourseMode.DEFAULT_MODE], modes)

    def test_nodes_for_course_single(self):
        """
        Find the modes for a course with only one mode
        """

        self.create_mode('verified', 'Verified Certificate')
        modes = CourseMode.modes_for_course(self.course_key)
        mode = Mode(u'verified', u'Verified Certificate', 0, '', 'usd', None)
        self.assertEqual([mode], modes)

        modes_dict = CourseMode.modes_for_course_dict(self.course_key)
        self.assertEqual(modes_dict['verified'], mode)
        self.assertEqual(CourseMode.mode_for_course(self.course_key, 'verified'),
                         mode)

    def test_modes_for_course_multiple(self):
        """
        Finding the modes when there's multiple modes
        """
        mode1 = Mode(u'honor', u'Honor Code Certificate', 0, '', 'usd', None)
        mode2 = Mode(u'verified', u'Verified Certificate', 0, '', 'usd', None)
        set_modes = [mode1, mode2]
        for mode in set_modes:
            self.create_mode(mode.slug, mode.name, mode.min_price, mode.suggested_prices)

        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual(modes, set_modes)
        self.assertEqual(mode1, CourseMode.mode_for_course(self.course_key, u'honor'))
        self.assertEqual(mode2, CourseMode.mode_for_course(self.course_key, u'verified'))
        self.assertIsNone(CourseMode.mode_for_course(self.course_key, 'DNE'))

    def test_min_course_price_for_currency(self):
        """
        Get the min course price for a course according to currency
        """
        # no modes, should get 0
        self.assertEqual(0, CourseMode.min_course_price_for_currency(self.course_key, 'usd'))

        # create some modes
        mode1 = Mode(u'honor', u'Honor Code Certificate', 10, '', 'usd', None)
        mode2 = Mode(u'verified', u'Verified Certificate', 20, '', 'usd', None)
        mode3 = Mode(u'honor', u'Honor Code Certificate', 80, '', 'cny', None)
        set_modes = [mode1, mode2, mode3]
        for mode in set_modes:
            self.create_mode(mode.slug, mode.name, mode.min_price, mode.suggested_prices, mode.currency)

        self.assertEqual(10, CourseMode.min_course_price_for_currency(self.course_key, 'usd'))
        self.assertEqual(80, CourseMode.min_course_price_for_currency(self.course_key, 'cny'))

    def test_modes_for_course_expired(self):
        expired_mode, _status = self.create_mode('verified', 'Verified Certificate')
        expired_mode.expiration_datetime = datetime.now(pytz.UTC) + timedelta(days=-1)
        expired_mode.save()
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([CourseMode.DEFAULT_MODE], modes)

        mode1 = Mode(u'honor', u'Honor Code Certificate', 0, '', 'usd', None)
        self.create_mode(mode1.slug, mode1.name, mode1.min_price, mode1.suggested_prices)
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([mode1], modes)

        expiration_datetime = datetime.now(pytz.UTC) + timedelta(days=1)
        expired_mode.expiration_datetime = expiration_datetime
        expired_mode.save()
        expired_mode_value = Mode(u'verified', u'Verified Certificate', 0, '', 'usd', expiration_datetime)
        modes = CourseMode.modes_for_course(self.course_key)
        self.assertEqual([expired_mode_value, mode1], modes)

        modes = CourseMode.modes_for_course(SlashSeparatedCourseKey('TestOrg', 'TestCourse', 'TestRun'))
        self.assertEqual([CourseMode.DEFAULT_MODE], modes)

########NEW FILE########
__FILENAME__ = urls
from django.conf.urls import include, patterns, url
from django.views.generic import TemplateView

from course_modes import views

urlpatterns = patterns(
    '',
    url(r'^choose/(?P<course_id>[^/]+/[^/]+/[^/]+)$', views.ChooseModeView.as_view(), name="course_modes_choose"),
)

########NEW FILE########
__FILENAME__ = views
"""
Views for the course_mode module
"""

import decimal
from django.core.urlresolvers import reverse
from django.http import (
    HttpResponseBadRequest,  Http404
)
from django.shortcuts import redirect
from django.views.generic.base import View
from django.utils.translation import ugettext as _
from django.contrib.auth.decorators import login_required
from django.utils.decorators import method_decorator

from edxmako.shortcuts import render_to_response

from course_modes.models import CourseMode
from courseware.access import has_access
from student.models import CourseEnrollment
from student.views import course_from_id
from verify_student.models import SoftwareSecurePhotoVerification
from xmodule.modulestore.locations import SlashSeparatedCourseKey


class ChooseModeView(View):
    """
    View used when the user is asked to pick a mode

    When a get request is used, shows the selection page.
    When a post request is used, assumes that it is a form submission
        from the selection page, parses the response, and then sends user
        to the next step in the flow
    """
    @method_decorator(login_required)
    def get(self, request, course_id, error=None):
        """ Displays the course mode choice page """

        course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)

        enrollment_mode = CourseEnrollment.enrollment_mode_for_user(request.user, course_key)
        upgrade = request.GET.get('upgrade', False)
        request.session['attempting_upgrade'] = upgrade

        # verified users do not need to register or upgrade
        if enrollment_mode == 'verified':
            return redirect(reverse('dashboard'))

        # registered users who are not trying to upgrade do not need to re-register
        if enrollment_mode is not None and upgrade is False:
            return redirect(reverse('dashboard'))

        modes = CourseMode.modes_for_course_dict(course_key)
        donation_for_course = request.session.get("donation_for_course", {})
        chosen_price = donation_for_course.get(course_key, None)

        course = course_from_id(course_key)
        context = {
            "course_modes_choose_url": reverse("course_modes_choose", kwargs={'course_id': course_key.to_deprecated_string()}),
            "modes": modes,
            "course_name": course.display_name_with_default,
            "course_org": course.display_org_with_default,
            "course_num": course.display_number_with_default,
            "chosen_price": chosen_price,
            "error": error,
            "upgrade": upgrade,
        }
        if "verified" in modes:
            context["suggested_prices"] = [decimal.Decimal(x) for x in modes["verified"].suggested_prices.split(",")]
            context["currency"] = modes["verified"].currency.upper()
            context["min_price"] = modes["verified"].min_price

        return render_to_response("course_modes/choose.html", context)

    @method_decorator(login_required)
    def post(self, request, course_id):
        """ Takes the form submission from the page and parses it """
        course_key = SlashSeparatedCourseKey.from_deprecated_string(course_id)
        user = request.user

        # This is a bit redundant with logic in student.views.change_enrollement,
        # but I don't really have the time to refactor it more nicely and test.
        course = course_from_id(course_key)
        if not has_access(user, 'enroll', course):
            error_msg = _("Enrollment is closed")
            return self.get(request, course_id, error=error_msg)

        upgrade = request.GET.get('upgrade', False)

        requested_mode = self.get_requested_mode(request.POST)

        allowed_modes = CourseMode.modes_for_course_dict(course_key)
        if requested_mode not in allowed_modes:
            return HttpResponseBadRequest(_("Enrollment mode not supported"))

        if requested_mode in ("audit", "honor"):
            CourseEnrollment.enroll(user, course_key, requested_mode)
            return redirect('dashboard')

        mode_info = allowed_modes[requested_mode]

        if requested_mode == "verified":
            amount = request.POST.get("contribution") or \
                request.POST.get("contribution-other-amt") or 0

            try:
                # validate the amount passed in and force it into two digits
                amount_value = decimal.Decimal(amount).quantize(decimal.Decimal('.01'), rounding=decimal.ROUND_DOWN)
            except decimal.InvalidOperation:
                error_msg = _("Invalid amount selected.")
                return self.get(request, course_id, error=error_msg)

            # Check for minimum pricing
            if amount_value < mode_info.min_price:
                error_msg = _("No selected price or selected price is too low.")
                return self.get(request, course_id, error=error_msg)

            donation_for_course = request.session.get("donation_for_course", {})
            donation_for_course[course_key] = amount_value
            request.session["donation_for_course"] = donation_for_course
            if SoftwareSecurePhotoVerification.user_has_valid_or_pending(request.user):
                return redirect(
                    reverse('verify_student_verified',
                            kwargs={'course_id': course_key.to_deprecated_string()}) + "?upgrade={}".format(upgrade)
                )

            return redirect(
                reverse('verify_student_show_requirements',
                        kwargs={'course_id': course_key.to_deprecated_string()}) + "?upgrade={}".format(upgrade))

    def get_requested_mode(self, request_dict):
        """
        Given the request object of `user_choice`, return the
        corresponding course mode slug
        """
        if 'audit_mode' in request_dict:
            return 'audit'
        if 'certificate_mode' and request_dict.get("honor-code"):
            return 'honor'
        if 'certificate_mode' in request_dict:
            return 'verified'

########NEW FILE########
__FILENAME__ = admin
"""
Admin site bindings for dark_lang
"""

from django.contrib import admin

from config_models.admin import ConfigurationModelAdmin
from dark_lang.models import DarkLangConfig

admin.site.register(DarkLangConfig, ConfigurationModelAdmin)

########NEW FILE########
__FILENAME__ = middleware
"""
Middleware for dark-launching languages. These languages won't be used
when determining which translation to give a user based on their browser
header, but can be selected by setting the ``preview-lang`` query parameter
to the language code.

Adding the query parameter ``clear-lang`` will reset the language stored
in the user's session.

This middleware must be placed before the LocaleMiddleware, but after
the SessionMiddleware.
"""
from django.conf import settings

from django.utils.translation.trans_real import parse_accept_lang_header

from dark_lang.models import DarkLangConfig


def dark_parse_accept_lang_header(accept):
    '''
    The use of 'zh-cn' for 'Simplified Chinese' and 'zh-tw' for 'Traditional Chinese'
    are now deprecated, as discussed here: https://code.djangoproject.com/ticket/18419.
    The new language codes 'zh-hans' and 'zh-hant' are now used since django 1.7.
    Although majority of browsers still use the old language codes, some new browsers
    such as IE11 in Windows 8.1 start to use the new ones, which makes the current
    chinese translations of edX don't work properly under these browsers.
    This function can keep compatibility between the old and new language codes. If one
    day edX uses django 1.7 or higher, this function can be modified to support the old
    language codes until there are no browsers use them.
    '''
    browser_langs = parse_accept_lang_header(accept)
    django_langs = []
    for lang, priority in browser_langs:
        lang = CHINESE_LANGUAGE_CODE_MAP.get(lang.lower(), lang)
        django_langs.append((lang, priority))

    return django_langs

# If django 1.7 or higher is used, the right-side can be updated with new-style codes.
CHINESE_LANGUAGE_CODE_MAP = {
    # The following are the new-style language codes for chinese language
    'zh-hans': 'zh-CN',     # Chinese (Simplified),
    'zh-hans-cn': 'zh-CN',  # Chinese (Simplified, China)
    'zh-hans-sg': 'zh-CN',  # Chinese (Simplified, Singapore)
    'zh-hant': 'zh-TW',     # Chinese (Traditional)
    'zh-hant-hk': 'zh-TW',  # Chinese (Traditional, Hongkong)
    'zh-hant-mo': 'zh-TW',  # Chinese (Traditional, Macau)
    'zh-hant-tw': 'zh-TW',  # Chinese (Traditional, Taiwan)
    # The following are the old-style language codes that django does not recognize
    'zh-hk': 'zh-TW',       # Chinese (Traditional, Hongkong)
    'zh-mo': 'zh-TW',       # Chinese (Traditional, Macau)
    'zh-sg': 'zh-CN',       # Chinese (Simplified, Singapore)
}


class DarkLangMiddleware(object):
    """
    Middleware for dark-launching languages.

    This is configured by creating ``DarkLangConfig`` rows in the database,
    using the django admin site.
    """

    @property
    def released_langs(self):
        """
        Current list of released languages
        """
        language_options = DarkLangConfig.current().released_languages_list
        if settings.LANGUAGE_CODE not in language_options:
            language_options.append(settings.LANGUAGE_CODE)
        return language_options

    def process_request(self, request):
        """
        Prevent user from requesting un-released languages except by using the preview-lang query string.
        """
        if not DarkLangConfig.current().enabled:
            return

        self._clean_accept_headers(request)
        self._activate_preview_language(request)

    def _is_released(self, lang_code):
        """
        ``True`` iff one of the values in ``self.released_langs`` is a prefix of ``lang_code``.
        """
        return any(lang_code.lower().startswith(released_lang.lower()) for released_lang in self.released_langs)

    def _format_accept_value(self, lang, priority=1.0):
        """
        Formats lang and priority into a valid accept header fragment.
        """
        return "{};q={}".format(lang, priority)

    def _clean_accept_headers(self, request):
        """
        Remove any language that is not either in ``self.released_langs`` or
        a territory of one of those languages.
        """
        accept = request.META.get('HTTP_ACCEPT_LANGUAGE', None)
        if accept is None or accept == '*':
            return

        new_accept = ", ".join(
            self._format_accept_value(lang, priority)
            for lang, priority
            in dark_parse_accept_lang_header(accept)
            if self._is_released(lang)
        )

        request.META['HTTP_ACCEPT_LANGUAGE'] = new_accept

    def _activate_preview_language(self, request):
        """
        If the request has the get parameter ``preview-lang``,
        and that language doesn't appear in ``self.released_langs``,
        then set the session ``django_language`` to that language.
        """
        if 'clear-lang' in request.GET:
            if 'django_language' in request.session:
                del request.session['django_language']

        preview_lang = request.GET.get('preview-lang', None)

        if not preview_lang:
            return

        if preview_lang in self.released_langs:
            return

        request.session['django_language'] = preview_lang

########NEW FILE########
__FILENAME__ = 0001_initial
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models


class Migration(SchemaMigration):

    def forwards(self, orm):
        # Adding model 'DarkLangConfig'
        db.create_table('dark_lang_darklangconfig', (
            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
            ('change_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
            ('changed_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, on_delete=models.PROTECT)),
            ('enabled', self.gf('django.db.models.fields.BooleanField')(default=False)),
            ('released_languages', self.gf('django.db.models.fields.TextField')(blank=True)),
        ))
        db.send_create_signal('dark_lang', ['DarkLangConfig'])


    def backwards(self, orm):
        # Deleting model 'DarkLangConfig'
        db.delete_table('dark_lang_darklangconfig')


    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'dark_lang.darklangconfig': {
            'Meta': {'object_name': 'DarkLangConfig'},
            'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}),
            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'released_languages': ('django.db.models.fields.TextField', [], {'blank': 'True'})
        }
    }

    complete_apps = ['dark_lang']
########NEW FILE########
__FILENAME__ = 0002_enable_on_install
# -*- coding: utf-8 -*-
import datetime
from south.db import db
from south.v2 import DataMigration
from django.db import models

class Migration(DataMigration):

    def forwards(self, orm):
        """
        Enable DarkLang by default when it is installed, to prevent accidental
        release of testing languages.
        """
        orm.DarkLangConfig(enabled=True).save()

    def backwards(self, orm):
        "Write your backwards methods here."
        raise RuntimeError("Cannot reverse this migration.")

    models = {
        'auth.group': {
            'Meta': {'object_name': 'Group'},
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
        },
        'auth.permission': {
            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
        },
        'auth.user': {
            'Meta': {'object_name': 'User'},
            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
        },
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
        },
        'dark_lang.darklangconfig': {
            'Meta': {'object_name': 'DarkLangConfig'},
            'change_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
            'changed_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.PROTECT'}),
            'enabled': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
            'released_languages': ('django.db.models.fields.TextField', [], {'blank': 'True'})
        }
    }

    complete_apps = ['dark_lang']
    symmetrical = True

########NEW FILE########
__FILENAME__ = models
"""
Models for the dark-launching languages
"""
from django.db import models

from config_models.models import ConfigurationModel


class DarkLangConfig(ConfigurationModel):
    """
    Configuration for the dark_lang django app
    """
    released_languages = models.TextField(
        blank=True,
        help_text="A comma-separated list of language codes to release to the public."
    )

    @property
    def released_languages_list(self):
        """
        ``released_languages`` as a list of language codes.

        Example: ['it', 'de-at', 'es', 'pt-br']
        """
        if not self.released_languages.strip():  # pylint: disable=no-member
            return []

        languages = [lang.strip() for lang in self.released_languages.split(',')]  # pylint: disable=no-member
        # Put in alphabetical order
        languages.sort()
        return languages

########NEW FILE########
__FILENAME__ = tests
"""
Tests of DarkLangMiddleware
"""
from django.contrib.auth.models import User
from django.http import HttpRequest

from django.test import TestCase
from mock import Mock

from dark_lang.middleware import DarkLangMiddleware
from dark_lang.models import DarkLangConfig


UNSET = object()


def set_if_set(dct, key, value):
    """
    Sets ``key`` in ``dct`` to ``value``
    unless ``value`` is ``UNSET``
    """
    if value is not UNSET:
        dct[key] = value


class DarkLangMiddlewareTests(TestCase):
    """
    Tests of DarkLangMiddleware
    """
    def setUp(self):
        self.user = User()
        self.user.save()
        DarkLangConfig(
            released_languages='rel',
            changed_by=self.user,
            enabled=True
        ).save()

    def process_request(self, django_language=UNSET, accept=UNSET, preview_lang=UNSET, clear_lang=UNSET):
        """
        Build a request and then process it using the ``DarkLangMiddleware``.

        Args:
            django_language (str): The language code to set in request.session['django_language']
            accept (str): The accept header to set in request.META['HTTP_ACCEPT_LANGUAGE']
            preview_lang (str): The value to set in request.GET['preview_lang']
            clear_lang (str): The value to set in request.GET['clear_lang']
        """
        session = {}
        set_if_set(session, 'django_language', django_language)

        meta = {}
        set_if_set(meta, 'HTTP_ACCEPT_LANGUAGE', accept)

        get = {}
        set_if_set(get, 'preview-lang', preview_lang)
        set_if_set(get, 'clear-lang', clear_lang)

        request = Mock(
            spec=HttpRequest,
            session=session,
            META=meta,
            GET=get
        )
        self.assertIsNone(DarkLangMiddleware().process_request(request))
        return request

    def assertAcceptEquals(self, value, request):
        """
        Assert that the HTML_ACCEPT_LANGUAGE header in request
        is equal to value
        """
        self.assertEquals(
            value,
            request.META.get('HTTP_ACCEPT_LANGUAGE', UNSET)
        )

    def test_empty_accept(self):
        self.assertAcceptEquals(UNSET, self.process_request())
