# -*- encoding: utf-8 -*-
#
# Copyright © 2014-2016 eNovance
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import functools
import uuid

import numpy
from unittest import mock

from gnocchi import carbonara
from gnocchi import incoming
from gnocchi import indexer
from gnocchi.rest.aggregates import api
from gnocchi.rest.aggregates import exceptions
from gnocchi.rest.aggregates import processor
from gnocchi import storage
from gnocchi.tests import base


class EqNan(object):
    def __eq__(self, other):
        return numpy.isnan(other)


eq_nan = EqNan()


def datetime64(*args):
    return numpy.datetime64(datetime.datetime(*args))


class TestGrouper(base.BaseTestCase):
    def setUp(self):
        super(TestGrouper, self).setUp()
        self.body = {"operations": "mean cpu.util"}
        self.references = self.body['operations']

        self.attribute_filters = {"attribute1": "value_attribute_1"}

        self.group_by = ["attribute1", "attribute2", "attribute3"]
        self.sorts = self.group_by

        self.start = datetime.datetime.now()
        self.end = self.start + datetime.timedelta(hours=1)

        self.granularity = 3600

        self.need_overlap = None
        self.fill = None

        self.details = True

        self.grouper_test = api.Grouper(self.group_by, self.start, self.end,
                                        self.body, self.sorts,
                                        self.attribute_filters,
                                        self.references, self.granularity,
                                        self.need_overlap, self.fill,
                                        self.details)

        self.test_resource_1 = {
            'uuid': '30b51786-944f-427b-85df-ce5f462e58a4',
            'revision_start': self.start - datetime.timedelta(hours=10),
            'revision_end': self.start - datetime.timedelta(hours=9),
            "attribute1": "a", "attribute2": "a", "attribute3": "a"
        }
        self.test_resource_2 = {
            'uuid': '17002c42-2f30-45fb-b1f1-eb33f7345e89',
            'revision_start': self.start - datetime.timedelta(hours=7),
            'revision_end': self.start - datetime.timedelta(hours=5),
            "attribute1": None, "attribute2": "b", "attribute3": "b"
        }
        self.test_resource_3 = {
            'uuid': '8f6be3bb-d5ee-4d01-986b-c034386637b0',
            'revision_start': self.start - datetime.timedelta(hours=2),
            'revision_end': self.start - datetime.timedelta(hours=1),
            "attribute1": "c", "attribute2": None, "attribute3": None
        }
        self.test_resources = [self.test_resource_1, self.test_resource_2,
                               self.test_resource_3]

    def test_create_history_period_filter_no_start(self):
        self.grouper_test.start = None
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual(self.attribute_filters, return_of_method)

    def test_create_history_period_filter_no_end(self):
        self.grouper_test.end = None
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual(self.attribute_filters, return_of_method)

    def test_create_history_period_filter_no_attribute_filter(self):
        self.grouper_test.attr_filter = None
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual({"and": [
            {"<": {'revision_start': self.end}},
            {"or": [
                {">=": {'revision_end': self.start}},
                {"=": {'revision_end': None}}]}]},
            return_of_method)

    def test_create_history_period_filter(self):
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual({"and": [{"and": [
            {"<": {'revision_start': self.end}},
            {"or": [
                {">=": {'revision_end': self.start}},
                {"=": {'revision_end': None}}]}]}, self.attribute_filters]},
            return_of_method)

    def test_group_resources_with_all_attributes(self):
        response_of_method = self.grouper_test.group(self.test_resources)

        self.assertEqual("17002c42-2f30-45fb-b1f1-eb33f7345e89",
                         response_of_method[0].resources[0].get('uuid'))
        self.assertEqual("30b51786-944f-427b-85df-ce5f462e58a4",
                         response_of_method[1].resources[0].get('uuid'))
        self.assertEqual("8f6be3bb-d5ee-4d01-986b-c034386637b0",
                         response_of_method[2].resources[0].get('uuid'))


class TestAggregatedTimeseries(base.BaseTestCase):
    @staticmethod
    def _resample_and_merge(ts, agg_dict):
        """Helper method that mimics _compute_splits_operations workflow."""
        grouped = ts.group_serie(agg_dict['sampling'])
        existing = agg_dict.get('return')
        name = agg_dict.get("name")
        resource = None if name is None else mock.Mock(id=str(uuid.uuid4()))
        metric = mock.Mock(id=str(uuid.uuid4()), name=name)
        agg_dict['return'] = (
            processor.MetricReference(metric, "mean", resource),
            carbonara.AggregatedTimeSerie.from_grouped_serie(
                grouped,
                carbonara.Aggregation(agg_dict['agg'],
                                      agg_dict['sampling'],
                                      None)))
        if existing:
            existing[2].merge(agg_dict['return'][2])
            agg_dict['return'] = existing

    def test_aggregated_different_archive_no_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean', "name": "all"}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean', "name": "all"}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 9, 1, 4), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc2))

        dtfrom = datetime64(2014, 1, 1, 11, 0, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          operations=["aggregate", "mean", [
                              "metric", ["all", "mean"]]])

    def test_aggregated_different_archive_no_overlap2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = carbonara.AggregatedTimeSerie(
            carbonara.Aggregation('mean', numpy.timedelta64(60, 's'), None))

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))
        metric = mock.Mock(id=str(uuid.uuid4()))
        ref = processor.MetricReference(metric, "mean")
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], (ref, tsc2)],
                          operations=["aggregate", "mean",
                                      ["metric", tsc1['return'][0].lookup_key,
                                       ref.lookup_key]])

    def test_aggregated_different_archive_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        # NOTE(sileht): minute 8 is missing in both and
        # minute 7 in tsc2 too, but it looks like we have
        # enough point to do the aggregation
        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 0, 0), 4),
            (datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 2),
            (datetime64(2014, 1, 1, 12, 3, 0), 4),
            (datetime64(2014, 1, 1, 12, 4, 0), 2),
            (datetime64(2014, 1, 1, 12, 5, 0), 3),
            (datetime64(2014, 1, 1, 12, 6, 0), 4),
            (datetime64(2014, 1, 1, 12, 7, 0), 10),
            (datetime64(2014, 1, 1, 12, 9, 0), 2)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 4),
            (datetime64(2014, 1, 1, 12, 3, 0), 4),
            (datetime64(2014, 1, 1, 12, 4, 0), 6),
            (datetime64(2014, 1, 1, 12, 5, 0), 3),
            (datetime64(2014, 1, 1, 12, 6, 0), 6),
            (datetime64(2014, 1, 1, 12, 9, 0), 2),
            (datetime64(2014, 1, 1, 12, 11, 0), 2),
            (datetime64(2014, 1, 1, 12, 12, 0), 2)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        dtfrom = datetime64(2014, 1, 1, 12, 0, 0)
        dtto = datetime64(2014, 1, 1, 12, 10, 0)

        # By default we require 100% of point that overlap
        # so that fail
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          to_timestamp=dtto,
                          operations=["aggregate", "mean", [
                              "metric",
                              tsc1['return'][0].lookup_key,
                              tsc2['return'][0].lookup_key,
                          ]])

        # Retry with 80% and it works
        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto,
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key,
            ]],
            needed_percent_of_overlap=80.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60, 's'), 3.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60, 's'), 3.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60, 's'), 4.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60, 's'), 4.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60, 's'), 3.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60, 's'), 5.0),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60, 's'), 10.0),
            (datetime64(2014, 1, 1, 12, 9, 0),
             numpy.timedelta64(60, 's'), 2.0),
        ], list(output))

    def test_aggregated_different_archive_overlap_edge_missing1(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 5, 0), 2),
            (datetime64(2014, 1, 1, 12, 6, 0), 7),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4),
            (datetime64(2014, 1, 1, 12, 5, 0), 16),
            (datetime64(2014, 1, 1, 12, 6, 0), 12)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        # By default we require 100% of point that overlap
        # but we allow that the last datapoint is missing
        # of the precisest granularity
        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60, 's'), 33.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60, 's'), 5.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60, 's'), 18.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60, 's'), 19.0),
        ], list(output))

    def test_aggregated_different_archive_overlap_edge_missing2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 4),
                                     (datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(
                2014, 1, 1, 12, 3, 0
            ), numpy.timedelta64(60000000000, 'ns'), 4.0),
        ], list(output))

    def test_processor_with_random_holes_derived_boundaries(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1),
            (datetime64(2014, 1, 1, 12, 4, 0), 5),
            (datetime64(2014, 1, 1, 12, 6, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=50.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.5),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 7.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 24.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 4.5),
        ], list(output))

    def test_processor_derived_missing_boundary(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1),
            (datetime64(2014, 1, 1, 12, 4, 0), 5),
            (datetime64(2014, 1, 1, 12, 6, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0),
            needed_percent_of_overlap=50.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.5),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 7.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 24.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 4.5),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key,
            ]],
            to_timestamp=datetime64(2014, 1, 1, 12, 7, 0),
            needed_percent_of_overlap=50.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.5),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 7.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 24.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 4.5),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
        ], list(output))

    def test_aggregated_some_overlap_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', "name": "foo"}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', "name": "bar"}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 5, 0), 2),
            (datetime64(2014, 1, 1, 12, 6, 0), 7),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4),
            (datetime64(2014, 1, 1, 12, 5, 0), 16),
            (datetime64(2014, 1, 1, 12, 6, 0), 12)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.5),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 16.5),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.5),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["-", ["metric"] + tsc1['return'][0].lookup_key,
                        ["metric"] + tsc2['return'][0].lookup_key
                        ], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), -6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), -2.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), -13),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), -15),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), -3),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), -14),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), -5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 3),
        ], list(output))

    def test_aggregated_some_overlap_with_fill_null(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 5, 0), 2),
            (datetime64(2014, 1, 1, 12, 6, 0), 7),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4),
            (datetime64(2014, 1, 1, 12, 5, 0), 16),
            (datetime64(2014, 1, 1, 12, 6, 0), 12)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], fill='null')["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 13.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 16.5),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.0),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["-", ["metric"] + tsc1['return'][0].lookup_key,
                        ["metric"] + tsc2['return'][0].lookup_key
                        ], fill='null')["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), -15.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), -3.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), -14.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), -5.0),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
        ], list(output))

    def test_aggregate_no_points_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.5),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 16.5),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.5),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["-", ["metric"] + tsc1['return'][0].lookup_key,
                        ["metric"] + tsc2['return'][0].lookup_key
                        ], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), -6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), -2.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), -13),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), -15),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), -3),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 3),
        ], list(output))

    def test_aggregated_nominal(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsc12 = {'sampling': numpy.timedelta64(300, 's'),
                 'size': 6, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc12['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsc22 = {'sampling': numpy.timedelta64(300, 's'),
                 'size': 6, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc22['sampling'])

        def ts1_update(ts):
            self._resample_and_merge(ts, tsc1)
            self._resample_and_merge(ts, tsc12)

        def ts2_update(ts):
            self._resample_and_merge(ts, tsc2)
            self._resample_and_merge(ts, tsc22)

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 4), 4),
            (datetime64(2014, 1, 1, 11, 47, 34), 8),
            (datetime64(2014, 1, 1, 11, 50, 54), 50),
            (datetime64(2014, 1, 1, 11, 54, 45), 4),
            (datetime64(2014, 1, 1, 11, 56, 49), 4),
            (datetime64(2014, 1, 1, 11, 57, 22), 6),
            (datetime64(2014, 1, 1, 11, 58, 22), 5),
            (datetime64(2014, 1, 1, 12, 1, 4), 4),
            (datetime64(2014, 1, 1, 12, 1, 9), 7),
            (datetime64(2014, 1, 1, 12, 2, 1), 15),
            (datetime64(2014, 1, 1, 12, 2, 12), 1),
            (datetime64(2014, 1, 1, 12, 3, 0), 3),
            (datetime64(2014, 1, 1, 12, 4, 9), 7),
            (datetime64(2014, 1, 1, 12, 5, 1), 15),
            (datetime64(2014, 1, 1, 12, 5, 12), 1),
            (datetime64(2014, 1, 1, 12, 6, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=ts1_update)

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 4), 6),
            (datetime64(2014, 1, 1, 11, 47, 34), 5),
            (datetime64(2014, 1, 1, 11, 50, 54), 51),
            (datetime64(2014, 1, 1, 11, 54, 45), 5),
            (datetime64(2014, 1, 1, 11, 56, 49), 5),
            (datetime64(2014, 1, 1, 11, 57, 22), 7),
            (datetime64(2014, 1, 1, 11, 58, 22), 5),
            (datetime64(2014, 1, 1, 12, 1, 4), 5),
            (datetime64(2014, 1, 1, 12, 1, 9), 8),
            (datetime64(2014, 1, 1, 12, 2, 1), 10),
            (datetime64(2014, 1, 1, 12, 2, 12), 2),
            (datetime64(2014, 1, 1, 12, 3, 0), 6),
            (datetime64(2014, 1, 1, 12, 4, 9), 4),
            (datetime64(2014, 1, 1, 12, 5, 1), 10),
            (datetime64(2014, 1, 1, 12, 5, 12), 1),
            (datetime64(2014, 1, 1, 12, 6, 0), 1)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=ts2_update)
        output = processor.aggregated(
            [tsc1['return'], tsc12['return'], tsc2['return'], tsc22['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key, tsc12['return'][0].lookup_key,
                tsc2['return'][0].lookup_key, tsc22['return'][0].lookup_key
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 11, 45),
             numpy.timedelta64(300, 's'), 5.75),
            (datetime64(2014, 1, 1, 11, 50),
             numpy.timedelta64(300, 's'), 27.5),
            (datetime64(2014, 1, 1, 11, 55),
             numpy.timedelta64(300, 's'), 5.3333333333333339),
            (datetime64(2014, 1, 1, 12, 0),
             numpy.timedelta64(300, 's'), 6.0),
            (datetime64(2014, 1, 1, 12, 5),
             numpy.timedelta64(300, 's'), 5.1666666666666661),
            (numpy.datetime64('2014-01-01T11:46:00.000000000'),
             numpy.timedelta64(60, 's'),
             5.0),
            (numpy.datetime64('2014-01-01T11:47:00.000000000'),
             numpy.timedelta64(60, 's'),
             6.5),
            (numpy.datetime64('2014-01-01T11:50:00.000000000'),
             numpy.timedelta64(60, 's'),
             50.5),
            (datetime64(2014, 1, 1, 11, 54),
             numpy.timedelta64(60, 's'), 4.5),
            (datetime64(2014, 1, 1, 11, 56),
             numpy.timedelta64(60, 's'), 4.5),
            (datetime64(2014, 1, 1, 11, 57),
             numpy.timedelta64(60, 's'), 6.5),
            (datetime64(2014, 1, 1, 11, 58),
             numpy.timedelta64(60, 's'), 5.0),
            (datetime64(2014, 1, 1, 12, 1),
             numpy.timedelta64(60, 's'), 6.0),
            (datetime64(2014, 1, 1, 12, 2),
             numpy.timedelta64(60, 's'), 7.0),
            (datetime64(2014, 1, 1, 12, 3),
             numpy.timedelta64(60, 's'), 4.5),
            (datetime64(2014, 1, 1, 12, 4),
             numpy.timedelta64(60, 's'), 5.5),
            (datetime64(2014, 1, 1, 12, 5),
             numpy.timedelta64(60, 's'), 6.75),
            (datetime64(2014, 1, 1, 12, 6),
             numpy.timedelta64(60, 's'), 2.0),
        ], list(output))

    def test_aggregated_partial_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(1, 's'),
                'size': 86400, 'agg': 'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(1, 's'),
                'size': 60, 'agg': 'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2015, 12, 3, 13, 19, 15), 1),
            (datetime64(2015, 12, 3, 13, 20, 15), 1),
            (datetime64(2015, 12, 3, 13, 21, 15), 1),
            (datetime64(2015, 12, 3, 13, 22, 15), 1)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2015, 12, 3, 13, 21, 15), 10),
            (datetime64(2015, 12, 3, 13, 22, 15), 10),
            (datetime64(2015, 12, 3, 13, 23, 15), 10),
            (datetime64(2015, 12, 3, 13, 24, 15), 10)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]])["aggregated"]

        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
        ], list(output))

        dtfrom = datetime64(2015, 12, 3, 13, 17, 0)
        dtto = datetime64(2015, 12, 3, 13, 25, 0)

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto,
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 19, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 20, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 23, 15
            ), numpy.timedelta64(1, 's'), 10.0),
            (datetime64(
                2015, 12, 3, 13, 24, 15
            ), numpy.timedelta64(1, 's'), 10.0),
        ], list(output))

        # Check boundaries are set when overlap=0
        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
        ], list(output))

        # By default we require 100% of point that overlap
        # so that fail if from or to is set
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          to_timestamp=dtto,
                          operations=["aggregate", "sum", [
                              "metric",
                              tsc1['return'][0].lookup_key,
                              tsc2['return'][0].lookup_key
                          ]])
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          operations=["aggregate", "sum", [
                              "metric",
                              tsc1['return'][0].lookup_key,
                              tsc2['return'][0].lookup_key
                          ]])
        # Retry with 50% and it works
        output = processor.aggregated(
            [tsc1['return'], tsc2['return']], from_timestamp=dtfrom,
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=50.0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 19, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 20, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
        ], list(output))

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']], to_timestamp=dtto,
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=50.0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 23, 15
            ), numpy.timedelta64(1, 's'), 10.0),
            (datetime64(
                2015, 12, 3, 13, 24, 15
            ), numpy.timedelta64(1, 's'), 10.0),
        ], list(output))


class CrossMetricAggregated(base.TestCase):
    def setUp(self):
        super(CrossMetricAggregated, self).setUp()
        # A lot of tests wants a metric, create one
        self.metric, __ = self._create_metric()

    def test_get_measures_empty_metrics_no_overlap(self):
        references = [
            processor.MetricReference(indexer.Metric(
                uuid.uuid4(), self.archive_policies['low']), 'mean'),
            processor.MetricReference(indexer.Metric(
                uuid.uuid4(), self.archive_policies['low']), 'mean'),
        ]
        self.assertRaises(
            exceptions.UnAggregableTimeseries,
            processor.get_measures, self.storage, references,
            operations=["aggregate", "mean", [
                "metric", ["whatever", "mean"], ["everwhat", "mean"],
            ]])

    def test_get_measures_empty_metric_needed_overlap_zero(self):
        m_id = str(self.metric.id)
        result = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            operations=["metric", m_id, "mean"], needed_overlap=0)
        self.assertEqual({m_id: {"mean": []}}, result)

    def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.assertRaises(storage.AggregationDoesNotExist,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, 'last'),
                           processor.MetricReference(metric2, 'last')],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "last"],
                              [(metric2.id), "last"],
                          ]])

    def test_get_measures_unknown_granularity(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, "mean"),
                           processor.MetricReference(metric2, "mean")],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "mean"],
                              [str(metric2.id), "mean"],
                          ]],
                          granularities=[numpy.timedelta64(12345456, 'ms')])

    def test_add_and_get_measures_different_archives(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['no_granularity_match'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])

        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, "mean"),
                           processor.MetricReference(metric2, "mean")],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "mean"],
                              [str(metric2.id), "mean"],
                          ]])

    def test_add_and_get_measures(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 12.5),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 24.0)
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "max", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 39.75),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 39.75),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 69),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 23),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 44)
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 0))["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 24.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            to_timestamp=datetime64(2014, 1, 1, 12, 5, 0))["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 10),
            to_timestamp=datetime64(2014, 1, 1, 12, 10, 10))["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 10),
             numpy.timedelta64(5, 'm'), 24.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0),
            to_timestamp=datetime64(2014, 1, 1, 12, 0, 1))["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0),
            to_timestamp=datetime64(2014, 1, 1, 12, 0, 1),
            granularities=[numpy.timedelta64(5, 'm')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
        ], values)

    def test_add_and_get_measures_with_holes(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 31), 8),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 42),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 6),
            incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10), 2),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, 'mean'),
             processor.MetricReference(metric2, 'mean')],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 18.875),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 18.875),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 11.0),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 22.0)
        ], values)

    def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["resample", "mean", numpy.timedelta64(1, 'D'),
             ["metric",
              [str(self.metric.id), "mean"],
              [str(metric2.id), "mean"]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 39.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 4.75)]
            }
        }, values)

    def test_resample_minus_2_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["-", ["resample", "mean", numpy.timedelta64(1, 'D'),
                   ["metric",
                    [str(self.metric.id), "mean"],
                    [str(metric2.id), "mean"]]], 2],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 37.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 2.75)]
            }
        }, values)

    def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["-",
             2,
             ["resample", "mean", numpy.timedelta64(1, 'D'),
              ["metric",
               [str(self.metric.id), "mean"],
               [str(metric2.id), "mean"]]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), -37.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), -2.75)]
            }
        }, values)

    def test_rolling(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 15, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 15, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["/", ["rolling", "sum", 2,
                   ["metric", [str(self.metric.id), "mean"],
                    [str(metric2.id), "mean"]]], 2],
            granularities=[numpy.timedelta64(5, 'm')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 5, 0),
                          numpy.timedelta64(5, 'm'), 55.5),
                         (datetime64(2014, 1, 1, 12, 10, 0),
                          numpy.timedelta64(5, 'm'), 23),
                         (datetime64(2014, 1, 1, 12, 15, 0),
                          numpy.timedelta64(5, 'm'), 24)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 5, 0),
                          numpy.timedelta64(5, 'm'), 5.5),
                         (datetime64(2014, 1, 1, 12, 10, 0),
                          numpy.timedelta64(5, 'm'), 3),
                         (datetime64(2014, 1, 1, 12, 15, 0),
                          numpy.timedelta64(5, 'm'), 4)]
            }
        }, values)

    def test_binary_operator_with_two_references(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["*", ["metric", str(self.metric.id), "mean"],
                  ["metric", str(metric2.id), "mean"]],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 621),
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 84),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 16),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 176),
        ], values)

    def test_binary_operator_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["*", ["metric", str(self.metric.id), "mean"], 2],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), 138),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 84),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 8),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 88)]
        }}, values)

    def test_ternary_operator_clip_min_max_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["clip", ["metric", str(self.metric.id), "mean"], 5, 60],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), 60),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 42),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 5),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 44)]
        }}, values)

    def test_ternary_operator_clip_min_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["clip", ["metric", str(self.metric.id), "mean"], 50],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), 69),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 50),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 50),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 50)]
        }}, values)

    def test_binary_operator_ts_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["*", 2, ["metric", str(self.metric.id), "mean"]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                      numpy.timedelta64(1, 'h'), 138),
                     (datetime64(2014, 1, 1, 13, 0, 0),
                      numpy.timedelta64(1, 'h'), 84),
                     (datetime64(2014, 1, 1, 14, 0, 0),
                      numpy.timedelta64(1, 'h'), 8),
                     (datetime64(2014, 1, 1, 15, 0, 0),
                      numpy.timedelta64(1, 'h'), 88)]
        }}, values)

    def test_mix(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            [
                "rolling",
                "sum",
                2,
                ["*", ["metric", str(self.metric.id), "mean"],
                      ["metric", str(metric2.id), "mean"]],
            ],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 705),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 100),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 192),
        ], values)

    def test_bool(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            [
                "gt",
                [
                    "/",
                    [
                        "*",
                        ["*", ["metric", str(self.metric.id), "mean"],
                              ["metric", str(metric2.id), "mean"]],
                        100,
                    ],
                    1000
                ],
                10
            ],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 1),
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 0),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 0),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 1),
        ], values)

    def test_unary_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), -69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), -4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), -9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), -2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), -4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["abs", ["metric", [str(self.metric.id), "mean"],
                     [str(metric2.id), "mean"]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                          numpy.timedelta64(1, 'h'), 69),
                         (datetime64(2014, 1, 1, 13, 0, 0),
                          numpy.timedelta64(1, 'h'), 42),
                         (datetime64(2014, 1, 1, 14, 0, 0),
                          numpy.timedelta64(1, 'h'), 4),
                         (datetime64(2014, 1, 1, 15, 0, 0),
                          numpy.timedelta64(1, 'h'), 44)]},
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                          numpy.timedelta64(1, 'h'), 9),
                         (datetime64(2014, 1, 1, 13, 0, 0),
                          numpy.timedelta64(1, 'h'), 2),
                         (datetime64(2014, 1, 1, 14, 0, 0),
                          numpy.timedelta64(1, 'h'), 4),
                         (datetime64(2014, 1, 1, 15, 0, 0),
                          numpy.timedelta64(1, 'h'), 4),
                         ]}
        }, values)

    def test_max_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), -69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), -4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), -9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), -2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), -4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            [
                "max",
                ["metric", str(self.metric.id), "mean"],
                [
                    "*",
                    ["metric", str(metric2.id), "mean"],
                    2,
                ]
            ],
            granularities=[numpy.timedelta64(1, 'h')]
        )["aggregated"]

        self.assertEqual(
            [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), -18),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 42),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 8),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 44)
            ], values)

    def test_rateofchangesec_operator(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 0),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 10), 450),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 5), 900),
            incoming.Measure(datetime64(2014, 1, 1, 12, 15, 30), 1350),
        ])
        self.trigger_processing([self.metric])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean")],
            ["rateofchangesec", ["metric", str(self.metric.id), "mean"]],
            granularities=[numpy.timedelta64(5, 'm')],
        )

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                          numpy.timedelta64(5, 'm'), eq_nan),
                         (datetime64(2014, 1, 1, 12, 5, 0),
                          numpy.timedelta64(5, 'm'), 1.5),
                         (datetime64(2014, 1, 1, 12, 10, 0),
                          numpy.timedelta64(5, 'm'), 1.5),
                         (datetime64(2014, 1, 1, 12, 15, 0),
                          numpy.timedelta64(5, 'm'), 1.5),
                         ]}
        }, values)

67b192d4ad18950d4e2fb4e0bb7db33643b9a11f
<mask token>


class CrossMetricAggregated(base.TestCase):

    def setUp(self):
        super(CrossMetricAggregated, self).setUp()
        self.metric, __ = self._create_metric()

    def test_get_measures_empty_metrics_no_overlap(self):
        references = [processor.MetricReference(indexer.Metric(uuid.uuid4(),
            self.archive_policies['low']), 'mean'), processor.
            MetricReference(indexer.Metric(uuid.uuid4(), self.
            archive_policies['low']), 'mean')]
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, references, operations=['aggregate',
            'mean', ['metric', ['whatever', 'mean'], ['everwhat', 'mean']]])
    <mask token>

    def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(storage.AggregationDoesNotExist, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'last'), processor.MetricReference(metric2, 'last')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'last'], [metric2.id, 'last']]])

    def test_get_measures_unknown_granularity(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(12345456, 'ms')])

    def test_add_and_get_measures_different_archives(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies[
            'no_granularity_match'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]])
    <mask token>
    <mask token>

    def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['resample', 'mean', numpy.timedelta64(1,
            'D'), ['metric', [str(self.metric.id), 'mean'], [str(metric2.id
            ), 'mean']]], granularities=[numpy.timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 39.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 4.75)]}}, values)

    def test_resample_minus_2_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]], 2], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 2.75)]}}, values)

    def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', 2, ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]]], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), -37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), -2.75)]}}, values)

    def test_rolling(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 5, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 15, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['/', ['rolling', 'sum', 2, ['metric', [str
            (self.metric.id), 'mean'], [str(metric2.id), 'mean']]], 2],
            granularities=[numpy.timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 55.5), (datetime64(
            2014, 1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 23), (
            datetime64(2014, 1, 1, 12, 15, 0), numpy.timedelta64(5, 'm'), 
            24)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 5,
            0), numpy.timedelta64(5, 'm'), 5.5), (datetime64(2014, 1, 1, 12,
            10, 0), numpy.timedelta64(5, 'm'), 3), (datetime64(2014, 1, 1, 
            12, 15, 0), numpy.timedelta64(5, 'm'), 4)]}}, values)
    <mask token>

    def test_binary_operator_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', ['metric', str(
            self.metric.id), 'mean'], 2], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_ternary_operator_clip_min_max_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 5, 60], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 60), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 5),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}}, values)

    def test_ternary_operator_clip_min_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 50], granularities=[numpy.timedelta64
            (1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 50), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 50
            ), (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'),
            50)]}}, values)

    def test_binary_operator_ts_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', 2, ['metric', str(
            self.metric.id), 'mean']], granularities=[numpy.timedelta64(1,
            'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_mix(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['rolling', 'sum', 2, ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 13, 0, 0), numpy.
            timedelta64(1, 'h'), 705), (datetime64(2014, 1, 1, 14, 0, 0),
            numpy.timedelta64(1, 'h'), 100), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 192)], values)

    def test_bool(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['gt', ['/', ['*', ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']],
            100], 1000], 10], granularities=[numpy.timedelta64(1, 'h')])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 1), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 14, 0, 0
            ), numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 1)], values)
    <mask token>

    def test_max_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['max', ['metric', str(self.metric.id),
            'mean'], ['*', ['metric', str(metric2.id), 'mean'], 2]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), -18), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 42), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 8), (datetime64(2014, 1, 1, 15, 
            0, 0), numpy.timedelta64(1, 'h'), 44)], values)

    def test_rateofchangesec_operator(self):
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 0), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 10), 450), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 5), 900), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 30), 1350)])
        self.trigger_processing([self.metric])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['rateofchangesec', [
            'metric', str(self.metric.id), 'mean']], granularities=[numpy.
            timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(5, 'm'), eq_nan), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 
            1.5), (datetime64(2014, 1, 1, 12, 10, 0), numpy.timedelta64(5,
            'm'), 1.5), (datetime64(2014, 1, 1, 12, 15, 0), numpy.
            timedelta64(5, 'm'), 1.5)]}}, values)

<mask token>


class TestAggregatedTimeseries(base.BaseTestCase):
    <mask token>

    def test_aggregated_different_archive_no_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean', 'name': 'all'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean', 'name': 'all'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4)
            ], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 9, 1, 4), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        dtfrom = datetime64(2014, 1, 1, 11, 0, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, operations=['aggregate', 'mean', ['metric', ['all',
            'mean']]])
    <mask token>

    def test_aggregated_different_archive_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 4),
            (datetime64(2014, 1, 1, 12, 1, 0), 3), (datetime64(2014, 1, 1, 
            12, 2, 0), 2), (datetime64(2014, 1, 1, 12, 3, 0), 4), (
            datetime64(2014, 1, 1, 12, 4, 0), 2), (datetime64(2014, 1, 1, 
            12, 5, 0), 3), (datetime64(2014, 1, 1, 12, 6, 0), 4), (
            datetime64(2014, 1, 1, 12, 7, 0), 10), (datetime64(2014, 1, 1, 
            12, 9, 0), 2)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 4), (datetime64(2014, 1, 1, 
            12, 3, 0), 4), (datetime64(2014, 1, 1, 12, 4, 0), 6), (
            datetime64(2014, 1, 1, 12, 5, 0), 3), (datetime64(2014, 1, 1, 
            12, 6, 0), 6), (datetime64(2014, 1, 1, 12, 9, 0), 2), (
            datetime64(2014, 1, 1, 12, 11, 0), 2), (datetime64(2014, 1, 1, 
            12, 12, 0), 2)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        dtfrom = datetime64(2014, 1, 1, 12, 0, 0)
        dtto = datetime64(2014, 1, 1, 12, 10, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, to_timestamp=dtto, operations=['aggregate', 'mean', [
            'metric', tsc1['return'][0].lookup_key, tsc2['return'][0].
            lookup_key]])
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto, operations=[
            'aggregate', 'mean', ['metric', tsc1['return'][0].lookup_key,
            tsc2['return'][0].lookup_key]], needed_percent_of_overlap=80.0)[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60, 's'), 3.0), (datetime64(2014, 1, 1, 12, 2, 0),
            numpy.timedelta64(60, 's'), 3.0), (datetime64(2014, 1, 1, 12, 3,
            0), numpy.timedelta64(60, 's'), 4.0), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60, 's'), 4.0), (datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(60, 's'), 3.0), (datetime64(
            2014, 1, 1, 12, 6, 0), numpy.timedelta64(60, 's'), 5.0), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60, 's'), 
            10.0), (datetime64(2014, 1, 1, 12, 9, 0), numpy.timedelta64(60,
            's'), 2.0)], list(output))
    <mask token>
    <mask token>

    def test_processor_with_random_holes_derived_boundaries(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1), (datetime64(2014, 1, 1, 
            12, 4, 0), 5), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60000000000, 'ns'), 5.5), (datetime64(2014, 1, 1, 
            12, 2, 0), numpy.timedelta64(60000000000, 'ns'), 7.0), (
            datetime64(2014, 1, 1, 12, 3, 0), numpy.timedelta64(60000000000,
            'ns'), 24.0), (datetime64(2014, 1, 1, 12, 4, 0), numpy.
            timedelta64(60000000000, 'ns'), 4.5)], list(output))
    <mask token>
    <mask token>
    <mask token>

    def test_aggregate_no_points_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 7, 0), 5), (datetime64(2014, 1, 1, 12, 8, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], fill=0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), 3.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), 1.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), 6.5), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), 16.5), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), 2.5), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60000000000,
            'ns'), 2.5), (datetime64(2014, 1, 1, 12, 8, 0), numpy.
            timedelta64(60000000000, 'ns'), 1.5)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['-', ['metric'] + tsc1['return'][0].lookup_key, [
            'metric'] + tsc2['return'][0].lookup_key], fill=0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), -6.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), -2.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), -13), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), -15), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), -3), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60000000000,
            'ns'), 5), (datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64
            (60000000000, 'ns'), 3)], list(output))

    def test_aggregated_nominal(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsc12 = {'sampling': numpy.timedelta64(300, 's'), 'size': 6, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc12['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsc22 = {'sampling': numpy.timedelta64(300, 's'), 'size': 6, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc22['sampling'])

        def ts1_update(ts):
            self._resample_and_merge(ts, tsc1)
            self._resample_and_merge(ts, tsc12)

        def ts2_update(ts):
            self._resample_and_merge(ts, tsc2)
            self._resample_and_merge(ts, tsc22)
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4),
            (datetime64(2014, 1, 1, 11, 47, 34), 8), (datetime64(2014, 1, 1,
            11, 50, 54), 50), (datetime64(2014, 1, 1, 11, 54, 45), 4), (
            datetime64(2014, 1, 1, 11, 56, 49), 4), (datetime64(2014, 1, 1,
            11, 57, 22), 6), (datetime64(2014, 1, 1, 11, 58, 22), 5), (
            datetime64(2014, 1, 1, 12, 1, 4), 4), (datetime64(2014, 1, 1, 
            12, 1, 9), 7), (datetime64(2014, 1, 1, 12, 2, 1), 15), (
            datetime64(2014, 1, 1, 12, 2, 12), 1), (datetime64(2014, 1, 1, 
            12, 3, 0), 3), (datetime64(2014, 1, 1, 12, 4, 9), 7), (
            datetime64(2014, 1, 1, 12, 5, 1), 15), (datetime64(2014, 1, 1, 
            12, 5, 12), 1), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            ts1_update)
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 6),
            (datetime64(2014, 1, 1, 11, 47, 34), 5), (datetime64(2014, 1, 1,
            11, 50, 54), 51), (datetime64(2014, 1, 1, 11, 54, 45), 5), (
            datetime64(2014, 1, 1, 11, 56, 49), 5), (datetime64(2014, 1, 1,
            11, 57, 22), 7), (datetime64(2014, 1, 1, 11, 58, 22), 5), (
            datetime64(2014, 1, 1, 12, 1, 4), 5), (datetime64(2014, 1, 1, 
            12, 1, 9), 8), (datetime64(2014, 1, 1, 12, 2, 1), 10), (
            datetime64(2014, 1, 1, 12, 2, 12), 2), (datetime64(2014, 1, 1, 
            12, 3, 0), 6), (datetime64(2014, 1, 1, 12, 4, 9), 4), (
            datetime64(2014, 1, 1, 12, 5, 1), 10), (datetime64(2014, 1, 1, 
            12, 5, 12), 1), (datetime64(2014, 1, 1, 12, 6, 0), 1)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            ts2_update)
        output = processor.aggregated([tsc1['return'], tsc12['return'],
            tsc2['return'], tsc22['return']], operations=['aggregate',
            'mean', ['metric', tsc1['return'][0].lookup_key, tsc12['return'
            ][0].lookup_key, tsc2['return'][0].lookup_key, tsc22['return'][
            0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 11, 45), numpy.
            timedelta64(300, 's'), 5.75), (datetime64(2014, 1, 1, 11, 50),
            numpy.timedelta64(300, 's'), 27.5), (datetime64(2014, 1, 1, 11,
            55), numpy.timedelta64(300, 's'), 5.333333333333334), (
            datetime64(2014, 1, 1, 12, 0), numpy.timedelta64(300, 's'), 6.0
            ), (datetime64(2014, 1, 1, 12, 5), numpy.timedelta64(300, 's'),
            5.166666666666666), (numpy.datetime64(
            '2014-01-01T11:46:00.000000000'), numpy.timedelta64(60, 's'), 
            5.0), (numpy.datetime64('2014-01-01T11:47:00.000000000'), numpy
            .timedelta64(60, 's'), 6.5), (numpy.datetime64(
            '2014-01-01T11:50:00.000000000'), numpy.timedelta64(60, 's'), 
            50.5), (datetime64(2014, 1, 1, 11, 54), numpy.timedelta64(60,
            's'), 4.5), (datetime64(2014, 1, 1, 11, 56), numpy.timedelta64(
            60, 's'), 4.5), (datetime64(2014, 1, 1, 11, 57), numpy.
            timedelta64(60, 's'), 6.5), (datetime64(2014, 1, 1, 11, 58),
            numpy.timedelta64(60, 's'), 5.0), (datetime64(2014, 1, 1, 12, 1
            ), numpy.timedelta64(60, 's'), 6.0), (datetime64(2014, 1, 1, 12,
            2), numpy.timedelta64(60, 's'), 7.0), (datetime64(2014, 1, 1, 
            12, 3), numpy.timedelta64(60, 's'), 4.5), (datetime64(2014, 1, 
            1, 12, 4), numpy.timedelta64(60, 's'), 5.5), (datetime64(2014, 
            1, 1, 12, 5), numpy.timedelta64(60, 's'), 6.75), (datetime64(
            2014, 1, 1, 12, 6), numpy.timedelta64(60, 's'), 2.0)], list(output)
            )

    def test_aggregated_partial_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(1, 's'), 'size': 86400, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(1, 's'), 'size': 60, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2015, 12, 3, 13, 19, 15), 
            1), (datetime64(2015, 12, 3, 13, 20, 15), 1), (datetime64(2015,
            12, 3, 13, 21, 15), 1), (datetime64(2015, 12, 3, 13, 22, 15), 1
            )], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2015, 12, 3, 13, 21, 15), 
            10), (datetime64(2015, 12, 3, 13, 22, 15), 10), (datetime64(
            2015, 12, 3, 13, 23, 15), 10), (datetime64(2015, 12, 3, 13, 24,
            15), 10)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0)], list(output))
        dtfrom = datetime64(2015, 12, 3, 13, 17, 0)
        dtto = datetime64(2015, 12, 3, 13, 25, 0)
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto, operations=[
            'aggregate', 'sum', ['metric', tsc1['return'][0].lookup_key,
            tsc2['return'][0].lookup_key]], needed_percent_of_overlap=0)[
            'aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 19, 15), numpy.
            timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 20, 15),
            numpy.timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 
            21, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12,
            3, 13, 22, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(
            2015, 12, 3, 13, 23, 15), numpy.timedelta64(1, 's'), 10.0), (
            datetime64(2015, 12, 3, 13, 24, 15), numpy.timedelta64(1, 's'),
            10.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0)], list(output))
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], to_timestamp=dtto,
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, operations=['aggregate', 'sum', ['metric', tsc1[
            'return'][0].lookup_key, tsc2['return'][0].lookup_key]])
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, operations=['aggregate', 'sum', [
            'metric', tsc1['return'][0].lookup_key, tsc2['return'][0].
            lookup_key]], needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 19, 15), numpy.
            timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 20, 15),
            numpy.timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 
            21, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12,
            3, 13, 22, 15), numpy.timedelta64(1, 's'), 11.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            to_timestamp=dtto, operations=['aggregate', 'sum', ['metric',
            tsc1['return'][0].lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 
            13, 23, 15), numpy.timedelta64(1, 's'), 10.0), (datetime64(2015,
            12, 3, 13, 24, 15), numpy.timedelta64(1, 's'), 10.0)], list(output)
            )


class CrossMetricAggregated(base.TestCase):

    def setUp(self):
        super(CrossMetricAggregated, self).setUp()
        self.metric, __ = self._create_metric()

    def test_get_measures_empty_metrics_no_overlap(self):
        references = [processor.MetricReference(indexer.Metric(uuid.uuid4(),
            self.archive_policies['low']), 'mean'), processor.
            MetricReference(indexer.Metric(uuid.uuid4(), self.
            archive_policies['low']), 'mean')]
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, references, operations=['aggregate',
            'mean', ['metric', ['whatever', 'mean'], ['everwhat', 'mean']]])

    def test_get_measures_empty_metric_needed_overlap_zero(self):
        m_id = str(self.metric.id)
        result = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], operations=['metric',
            m_id, 'mean'], needed_overlap=0)
        self.assertEqual({m_id: {'mean': []}}, result)

    def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(storage.AggregationDoesNotExist, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'last'), processor.MetricReference(metric2, 'last')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'last'], [metric2.id, 'last']]])

    def test_get_measures_unknown_granularity(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(12345456, 'ms')])

    def test_add_and_get_measures_different_archives(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies[
            'no_granularity_match'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]])

    def test_add_and_get_measures(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 7, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 39.0), (datetime64(2014, 1, 1,
            12, 5, 0), numpy.timedelta64(5, 'm'), 12.5), (datetime64(2014, 
            1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'max', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 39.75), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 39.75), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 69), (datetime64(2014, 1, 1, 
            12, 5, 0), numpy.timedelta64(5, 'm'), 23), (datetime64(2014, 1,
            1, 12, 10, 0), numpy.timedelta64(5, 'm'), 44)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 0))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12), numpy.timedelta64(1,
            'h'), 22.25), (datetime64(2014, 1, 1, 12, 10, 0), numpy.
            timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            to_timestamp=datetime64(2014, 1, 1, 12, 5, 0))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 39.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 10), to_timestamp
            =datetime64(2014, 1, 1, 12, 10, 10))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12), numpy.timedelta64(1,
            'h'), 22.25), (datetime64(2014, 1, 1, 12, 10), numpy.
            timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0), to_timestamp=
            datetime64(2014, 1, 1, 12, 0, 1))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12, 0, 0), numpy.timedelta64
            (1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(5, 'm'), 39.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0), to_timestamp=
            datetime64(2014, 1, 1, 12, 0, 1), granularities=[numpy.
            timedelta64(5, 'm')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(5, 'm'), 39.0)], values)

    def test_add_and_get_measures_with_holes(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 8), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 42)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 7, 31), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            9, 31), 6), incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10),
            2)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 18.875), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 18.875), (datetime64(2014, 1, 1, 12,
            0, 0), numpy.timedelta64(5, 'm'), 39.0), (datetime64(2014, 1, 1,
            12, 5, 0), numpy.timedelta64(5, 'm'), 11.0), (datetime64(2014, 
            1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 22.0)], values)

    def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['resample', 'mean', numpy.timedelta64(1,
            'D'), ['metric', [str(self.metric.id), 'mean'], [str(metric2.id
            ), 'mean']]], granularities=[numpy.timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 39.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 4.75)]}}, values)

    def test_resample_minus_2_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]], 2], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 2.75)]}}, values)

    def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', 2, ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]]], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), -37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), -2.75)]}}, values)

    def test_rolling(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 5, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 15, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['/', ['rolling', 'sum', 2, ['metric', [str
            (self.metric.id), 'mean'], [str(metric2.id), 'mean']]], 2],
            granularities=[numpy.timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 55.5), (datetime64(
            2014, 1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 23), (
            datetime64(2014, 1, 1, 12, 15, 0), numpy.timedelta64(5, 'm'), 
            24)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 5,
            0), numpy.timedelta64(5, 'm'), 5.5), (datetime64(2014, 1, 1, 12,
            10, 0), numpy.timedelta64(5, 'm'), 3), (datetime64(2014, 1, 1, 
            12, 15, 0), numpy.timedelta64(5, 'm'), 4)]}}, values)

    def test_binary_operator_with_two_references(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['*', ['metric', str(self.metric.id),
            'mean'], ['metric', str(metric2.id), 'mean']], granularities=[
            numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 621), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 84), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 16), (datetime64(2014, 1, 1, 15,
            0, 0), numpy.timedelta64(1, 'h'), 176)], values)

    def test_binary_operator_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', ['metric', str(
            self.metric.id), 'mean'], 2], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_ternary_operator_clip_min_max_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 5, 60], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 60), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 5),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}}, values)

    def test_ternary_operator_clip_min_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 50], granularities=[numpy.timedelta64
            (1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 50), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 50
            ), (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'),
            50)]}}, values)

    def test_binary_operator_ts_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', 2, ['metric', str(
            self.metric.id), 'mean']], granularities=[numpy.timedelta64(1,
            'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_mix(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['rolling', 'sum', 2, ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 13, 0, 0), numpy.
            timedelta64(1, 'h'), 705), (datetime64(2014, 1, 1, 14, 0, 0),
            numpy.timedelta64(1, 'h'), 100), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 192)], values)

    def test_bool(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['gt', ['/', ['*', ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']],
            100], 1000], 10], granularities=[numpy.timedelta64(1, 'h')])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 1), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 14, 0, 0
            ), numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 1)], values)

    def test_unary_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['abs', ['metric', [str(self.metric.id),
            'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 4),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 0,
            0), numpy.timedelta64(1, 'h'), 9), (datetime64(2014, 1, 1, 13, 
            0, 0), numpy.timedelta64(1, 'h'), 2), (datetime64(2014, 1, 1, 
            14, 0, 0), numpy.timedelta64(1, 'h'), 4), (datetime64(2014, 1, 
            1, 15, 0, 0), numpy.timedelta64(1, 'h'), 4)]}}, values)

    def test_max_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['max', ['metric', str(self.metric.id),
            'mean'], ['*', ['metric', str(metric2.id), 'mean'], 2]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), -18), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 42), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 8), (datetime64(2014, 1, 1, 15, 
            0, 0), numpy.timedelta64(1, 'h'), 44)], values)

    def test_rateofchangesec_operator(self):
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 0), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 10), 450), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 5), 900), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 30), 1350)])
        self.trigger_processing([self.metric])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['rateofchangesec', [
            'metric', str(self.metric.id), 'mean']], granularities=[numpy.
            timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(5, 'm'), eq_nan), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 
            1.5), (datetime64(2014, 1, 1, 12, 10, 0), numpy.timedelta64(5,
            'm'), 1.5), (datetime64(2014, 1, 1, 12, 15, 0), numpy.
            timedelta64(5, 'm'), 1.5)]}}, values)

<mask token>


class TestAggregatedTimeseries(base.BaseTestCase):

    @staticmethod
    def _resample_and_merge(ts, agg_dict):
        """Helper method that mimics _compute_splits_operations workflow."""
        grouped = ts.group_serie(agg_dict['sampling'])
        existing = agg_dict.get('return')
        name = agg_dict.get('name')
        resource = None if name is None else mock.Mock(id=str(uuid.uuid4()))
        metric = mock.Mock(id=str(uuid.uuid4()), name=name)
        agg_dict['return'] = processor.MetricReference(metric, 'mean', resource
            ), carbonara.AggregatedTimeSerie.from_grouped_serie(grouped,
            carbonara.Aggregation(agg_dict['agg'], agg_dict['sampling'], None))
        if existing:
            existing[2].merge(agg_dict['return'][2])
            agg_dict['return'] = existing

    def test_aggregated_different_archive_no_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean', 'name': 'all'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean', 'name': 'all'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4)
            ], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 9, 1, 4), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        dtfrom = datetime64(2014, 1, 1, 11, 0, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, operations=['aggregate', 'mean', ['metric', ['all',
            'mean']]])

    def test_aggregated_different_archive_no_overlap2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = carbonara.AggregatedTimeSerie(carbonara.Aggregation('mean',
            numpy.timedelta64(60, 's'), None))
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        metric = mock.Mock(id=str(uuid.uuid4()))
        ref = processor.MetricReference(metric, 'mean')
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], (ref, tsc2)], operations=[
            'aggregate', 'mean', ['metric', tsc1['return'][0].lookup_key,
            ref.lookup_key]])

    def test_aggregated_different_archive_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 4),
            (datetime64(2014, 1, 1, 12, 1, 0), 3), (datetime64(2014, 1, 1, 
            12, 2, 0), 2), (datetime64(2014, 1, 1, 12, 3, 0), 4), (
            datetime64(2014, 1, 1, 12, 4, 0), 2), (datetime64(2014, 1, 1, 
            12, 5, 0), 3), (datetime64(2014, 1, 1, 12, 6, 0), 4), (
            datetime64(2014, 1, 1, 12, 7, 0), 10), (datetime64(2014, 1, 1, 
            12, 9, 0), 2)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 4), (datetime64(2014, 1, 1, 
            12, 3, 0), 4), (datetime64(2014, 1, 1, 12, 4, 0), 6), (
            datetime64(2014, 1, 1, 12, 5, 0), 3), (datetime64(2014, 1, 1, 
            12, 6, 0), 6), (datetime64(2014, 1, 1, 12, 9, 0), 2), (
            datetime64(2014, 1, 1, 12, 11, 0), 2), (datetime64(2014, 1, 1, 
            12, 12, 0), 2)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        dtfrom = datetime64(2014, 1, 1, 12, 0, 0)
        dtto = datetime64(2014, 1, 1, 12, 10, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, to_timestamp=dtto, operations=['aggregate', 'mean', [
            'metric', tsc1['return'][0].lookup_key, tsc2['return'][0].
            lookup_key]])
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto, operations=[
            'aggregate', 'mean', ['metric', tsc1['return'][0].lookup_key,
            tsc2['return'][0].lookup_key]], needed_percent_of_overlap=80.0)[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60, 's'), 3.0), (datetime64(2014, 1, 1, 12, 2, 0),
            numpy.timedelta64(60, 's'), 3.0), (datetime64(2014, 1, 1, 12, 3,
            0), numpy.timedelta64(60, 's'), 4.0), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60, 's'), 4.0), (datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(60, 's'), 3.0), (datetime64(
            2014, 1, 1, 12, 6, 0), numpy.timedelta64(60, 's'), 5.0), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60, 's'), 
            10.0), (datetime64(2014, 1, 1, 12, 9, 0), numpy.timedelta64(60,
            's'), 2.0)], list(output))

    def test_aggregated_different_archive_overlap_edge_missing1(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 5, 0), 2), (datetime64(2014, 1, 1, 12, 6, 0), 7), (
            datetime64(2014, 1, 1, 12, 7, 0), 5), (datetime64(2014, 1, 1, 
            12, 8, 0), 3)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4), (datetime64(2014, 1, 1, 
            12, 5, 0), 16), (datetime64(2014, 1, 1, 12, 6, 0), 12)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60, 's'), 33.0), (datetime64(2014, 1, 1, 12, 4, 0),
            numpy.timedelta64(60, 's'), 5.0), (datetime64(2014, 1, 1, 12, 5,
            0), numpy.timedelta64(60, 's'), 18.0), (datetime64(2014, 1, 1, 
            12, 6, 0), numpy.timedelta64(60, 's'), 19.0)], list(output))
    <mask token>

    def test_processor_with_random_holes_derived_boundaries(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1), (datetime64(2014, 1, 1, 
            12, 4, 0), 5), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60000000000, 'ns'), 5.5), (datetime64(2014, 1, 1, 
            12, 2, 0), numpy.timedelta64(60000000000, 'ns'), 7.0), (
            datetime64(2014, 1, 1, 12, 3, 0), numpy.timedelta64(60000000000,
            'ns'), 24.0), (datetime64(2014, 1, 1, 12, 4, 0), numpy.
            timedelta64(60000000000, 'ns'), 4.5)], list(output))
    <mask token>
    <mask token>

    def test_aggregated_some_overlap_with_fill_null(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 5, 0), 2), (datetime64(2014, 1, 1, 12, 6, 0), 7), (
            datetime64(2014, 1, 1, 12, 7, 0), 5), (datetime64(2014, 1, 1, 
            12, 8, 0), 3)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4), (datetime64(2014, 1, 1, 
            12, 5, 0), 16), (datetime64(2014, 1, 1, 12, 6, 0), 12)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], fill='null')[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), 6.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), 2.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), 13.0), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), 16.5), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), 2.5), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(60000000000,
            'ns'), 9.0), (datetime64(2014, 1, 1, 12, 6, 0), numpy.
            timedelta64(60000000000, 'ns'), 9.5), (datetime64(2014, 1, 1, 
            12, 7, 0), numpy.timedelta64(60000000000, 'ns'), 5.0), (
            datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64(60000000000,
            'ns'), 3.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['-', ['metric'] + tsc1['return'][0].lookup_key, [
            'metric'] + tsc2['return'][0].lookup_key], fill='null')[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), eq_nan), (datetime64(2014, 1, 1,
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), eq_nan), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), eq_nan), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), -15.0), (datetime64(2014, 1, 1,
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), -3.0), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(60000000000,
            'ns'), -14.0), (datetime64(2014, 1, 1, 12, 6, 0), numpy.
            timedelta64(60000000000, 'ns'), -5.0), (datetime64(2014, 1, 1, 
            12, 7, 0), numpy.timedelta64(60000000000, 'ns'), eq_nan), (
            datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64(60000000000,
            'ns'), eq_nan)], list(output))

    def test_aggregate_no_points_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 7, 0), 5), (datetime64(2014, 1, 1, 12, 8, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], fill=0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), 3.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), 1.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), 6.5), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), 16.5), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), 2.5), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60000000000,
            'ns'), 2.5), (datetime64(2014, 1, 1, 12, 8, 0), numpy.
            timedelta64(60000000000, 'ns'), 1.5)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['-', ['metric'] + tsc1['return'][0].lookup_key, [
            'metric'] + tsc2['return'][0].lookup_key], fill=0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), -6.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), -2.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), -13), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), -15), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), -3), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60000000000,
            'ns'), 5), (datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64
            (60000000000, 'ns'), 3)], list(output))

    def test_aggregated_nominal(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsc12 = {'sampling': numpy.timedelta64(300, 's'), 'size': 6, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc12['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsc22 = {'sampling': numpy.timedelta64(300, 's'), 'size': 6, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc22['sampling'])

        def ts1_update(ts):
            self._resample_and_merge(ts, tsc1)
            self._resample_and_merge(ts, tsc12)

        def ts2_update(ts):
            self._resample_and_merge(ts, tsc2)
            self._resample_and_merge(ts, tsc22)
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4),
            (datetime64(2014, 1, 1, 11, 47, 34), 8), (datetime64(2014, 1, 1,
            11, 50, 54), 50), (datetime64(2014, 1, 1, 11, 54, 45), 4), (
            datetime64(2014, 1, 1, 11, 56, 49), 4), (datetime64(2014, 1, 1,
            11, 57, 22), 6), (datetime64(2014, 1, 1, 11, 58, 22), 5), (
            datetime64(2014, 1, 1, 12, 1, 4), 4), (datetime64(2014, 1, 1, 
            12, 1, 9), 7), (datetime64(2014, 1, 1, 12, 2, 1), 15), (
            datetime64(2014, 1, 1, 12, 2, 12), 1), (datetime64(2014, 1, 1, 
            12, 3, 0), 3), (datetime64(2014, 1, 1, 12, 4, 9), 7), (
            datetime64(2014, 1, 1, 12, 5, 1), 15), (datetime64(2014, 1, 1, 
            12, 5, 12), 1), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            ts1_update)
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 6),
            (datetime64(2014, 1, 1, 11, 47, 34), 5), (datetime64(2014, 1, 1,
            11, 50, 54), 51), (datetime64(2014, 1, 1, 11, 54, 45), 5), (
            datetime64(2014, 1, 1, 11, 56, 49), 5), (datetime64(2014, 1, 1,
            11, 57, 22), 7), (datetime64(2014, 1, 1, 11, 58, 22), 5), (
            datetime64(2014, 1, 1, 12, 1, 4), 5), (datetime64(2014, 1, 1, 
            12, 1, 9), 8), (datetime64(2014, 1, 1, 12, 2, 1), 10), (
            datetime64(2014, 1, 1, 12, 2, 12), 2), (datetime64(2014, 1, 1, 
            12, 3, 0), 6), (datetime64(2014, 1, 1, 12, 4, 9), 4), (
            datetime64(2014, 1, 1, 12, 5, 1), 10), (datetime64(2014, 1, 1, 
            12, 5, 12), 1), (datetime64(2014, 1, 1, 12, 6, 0), 1)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            ts2_update)
        output = processor.aggregated([tsc1['return'], tsc12['return'],
            tsc2['return'], tsc22['return']], operations=['aggregate',
            'mean', ['metric', tsc1['return'][0].lookup_key, tsc12['return'
            ][0].lookup_key, tsc2['return'][0].lookup_key, tsc22['return'][
            0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 11, 45), numpy.
            timedelta64(300, 's'), 5.75), (datetime64(2014, 1, 1, 11, 50),
            numpy.timedelta64(300, 's'), 27.5), (datetime64(2014, 1, 1, 11,
            55), numpy.timedelta64(300, 's'), 5.333333333333334), (
            datetime64(2014, 1, 1, 12, 0), numpy.timedelta64(300, 's'), 6.0
            ), (datetime64(2014, 1, 1, 12, 5), numpy.timedelta64(300, 's'),
            5.166666666666666), (numpy.datetime64(
            '2014-01-01T11:46:00.000000000'), numpy.timedelta64(60, 's'), 
            5.0), (numpy.datetime64('2014-01-01T11:47:00.000000000'), numpy
            .timedelta64(60, 's'), 6.5), (numpy.datetime64(
            '2014-01-01T11:50:00.000000000'), numpy.timedelta64(60, 's'), 
            50.5), (datetime64(2014, 1, 1, 11, 54), numpy.timedelta64(60,
            's'), 4.5), (datetime64(2014, 1, 1, 11, 56), numpy.timedelta64(
            60, 's'), 4.5), (datetime64(2014, 1, 1, 11, 57), numpy.
            timedelta64(60, 's'), 6.5), (datetime64(2014, 1, 1, 11, 58),
            numpy.timedelta64(60, 's'), 5.0), (datetime64(2014, 1, 1, 12, 1
            ), numpy.timedelta64(60, 's'), 6.0), (datetime64(2014, 1, 1, 12,
            2), numpy.timedelta64(60, 's'), 7.0), (datetime64(2014, 1, 1, 
            12, 3), numpy.timedelta64(60, 's'), 4.5), (datetime64(2014, 1, 
            1, 12, 4), numpy.timedelta64(60, 's'), 5.5), (datetime64(2014, 
            1, 1, 12, 5), numpy.timedelta64(60, 's'), 6.75), (datetime64(
            2014, 1, 1, 12, 6), numpy.timedelta64(60, 's'), 2.0)], list(output)
            )

    def test_aggregated_partial_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(1, 's'), 'size': 86400, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(1, 's'), 'size': 60, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2015, 12, 3, 13, 19, 15), 
            1), (datetime64(2015, 12, 3, 13, 20, 15), 1), (datetime64(2015,
            12, 3, 13, 21, 15), 1), (datetime64(2015, 12, 3, 13, 22, 15), 1
            )], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2015, 12, 3, 13, 21, 15), 
            10), (datetime64(2015, 12, 3, 13, 22, 15), 10), (datetime64(
            2015, 12, 3, 13, 23, 15), 10), (datetime64(2015, 12, 3, 13, 24,
            15), 10)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0)], list(output))
        dtfrom = datetime64(2015, 12, 3, 13, 17, 0)
        dtto = datetime64(2015, 12, 3, 13, 25, 0)
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto, operations=[
            'aggregate', 'sum', ['metric', tsc1['return'][0].lookup_key,
            tsc2['return'][0].lookup_key]], needed_percent_of_overlap=0)[
            'aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 19, 15), numpy.
            timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 20, 15),
            numpy.timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 
            21, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12,
            3, 13, 22, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(
            2015, 12, 3, 13, 23, 15), numpy.timedelta64(1, 's'), 10.0), (
            datetime64(2015, 12, 3, 13, 24, 15), numpy.timedelta64(1, 's'),
            10.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0)], list(output))
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], to_timestamp=dtto,
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, operations=['aggregate', 'sum', ['metric', tsc1[
            'return'][0].lookup_key, tsc2['return'][0].lookup_key]])
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, operations=['aggregate', 'sum', [
            'metric', tsc1['return'][0].lookup_key, tsc2['return'][0].
            lookup_key]], needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 19, 15), numpy.
            timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 20, 15),
            numpy.timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 
            21, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12,
            3, 13, 22, 15), numpy.timedelta64(1, 's'), 11.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            to_timestamp=dtto, operations=['aggregate', 'sum', ['metric',
            tsc1['return'][0].lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 
            13, 23, 15), numpy.timedelta64(1, 's'), 10.0), (datetime64(2015,
            12, 3, 13, 24, 15), numpy.timedelta64(1, 's'), 10.0)], list(output)
            )


class CrossMetricAggregated(base.TestCase):

    def setUp(self):
        super(CrossMetricAggregated, self).setUp()
        self.metric, __ = self._create_metric()

    def test_get_measures_empty_metrics_no_overlap(self):
        references = [processor.MetricReference(indexer.Metric(uuid.uuid4(),
            self.archive_policies['low']), 'mean'), processor.
            MetricReference(indexer.Metric(uuid.uuid4(), self.
            archive_policies['low']), 'mean')]
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, references, operations=['aggregate',
            'mean', ['metric', ['whatever', 'mean'], ['everwhat', 'mean']]])

    def test_get_measures_empty_metric_needed_overlap_zero(self):
        m_id = str(self.metric.id)
        result = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], operations=['metric',
            m_id, 'mean'], needed_overlap=0)
        self.assertEqual({m_id: {'mean': []}}, result)

    def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(storage.AggregationDoesNotExist, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'last'), processor.MetricReference(metric2, 'last')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'last'], [metric2.id, 'last']]])

    def test_get_measures_unknown_granularity(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(12345456, 'ms')])

    def test_add_and_get_measures_different_archives(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies[
            'no_granularity_match'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]])

    def test_add_and_get_measures(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 7, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 39.0), (datetime64(2014, 1, 1,
            12, 5, 0), numpy.timedelta64(5, 'm'), 12.5), (datetime64(2014, 
            1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'max', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 39.75), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 39.75), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 69), (datetime64(2014, 1, 1, 
            12, 5, 0), numpy.timedelta64(5, 'm'), 23), (datetime64(2014, 1,
            1, 12, 10, 0), numpy.timedelta64(5, 'm'), 44)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 0))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12), numpy.timedelta64(1,
            'h'), 22.25), (datetime64(2014, 1, 1, 12, 10, 0), numpy.
            timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            to_timestamp=datetime64(2014, 1, 1, 12, 5, 0))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 39.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 10), to_timestamp
            =datetime64(2014, 1, 1, 12, 10, 10))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12), numpy.timedelta64(1,
            'h'), 22.25), (datetime64(2014, 1, 1, 12, 10), numpy.
            timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0), to_timestamp=
            datetime64(2014, 1, 1, 12, 0, 1))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12, 0, 0), numpy.timedelta64
            (1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(5, 'm'), 39.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0), to_timestamp=
            datetime64(2014, 1, 1, 12, 0, 1), granularities=[numpy.
            timedelta64(5, 'm')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(5, 'm'), 39.0)], values)

    def test_add_and_get_measures_with_holes(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 8), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 42)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 7, 31), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            9, 31), 6), incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10),
            2)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 18.875), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 18.875), (datetime64(2014, 1, 1, 12,
            0, 0), numpy.timedelta64(5, 'm'), 39.0), (datetime64(2014, 1, 1,
            12, 5, 0), numpy.timedelta64(5, 'm'), 11.0), (datetime64(2014, 
            1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 22.0)], values)

    def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['resample', 'mean', numpy.timedelta64(1,
            'D'), ['metric', [str(self.metric.id), 'mean'], [str(metric2.id
            ), 'mean']]], granularities=[numpy.timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 39.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 4.75)]}}, values)

    def test_resample_minus_2_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]], 2], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 2.75)]}}, values)

    def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', 2, ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]]], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), -37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), -2.75)]}}, values)

    def test_rolling(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 5, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 15, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['/', ['rolling', 'sum', 2, ['metric', [str
            (self.metric.id), 'mean'], [str(metric2.id), 'mean']]], 2],
            granularities=[numpy.timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 55.5), (datetime64(
            2014, 1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 23), (
            datetime64(2014, 1, 1, 12, 15, 0), numpy.timedelta64(5, 'm'), 
            24)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 5,
            0), numpy.timedelta64(5, 'm'), 5.5), (datetime64(2014, 1, 1, 12,
            10, 0), numpy.timedelta64(5, 'm'), 3), (datetime64(2014, 1, 1, 
            12, 15, 0), numpy.timedelta64(5, 'm'), 4)]}}, values)

    def test_binary_operator_with_two_references(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['*', ['metric', str(self.metric.id),
            'mean'], ['metric', str(metric2.id), 'mean']], granularities=[
            numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 621), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 84), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 16), (datetime64(2014, 1, 1, 15,
            0, 0), numpy.timedelta64(1, 'h'), 176)], values)

    def test_binary_operator_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', ['metric', str(
            self.metric.id), 'mean'], 2], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_ternary_operator_clip_min_max_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 5, 60], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 60), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 5),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}}, values)

    def test_ternary_operator_clip_min_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 50], granularities=[numpy.timedelta64
            (1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 50), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 50
            ), (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'),
            50)]}}, values)

    def test_binary_operator_ts_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', 2, ['metric', str(
            self.metric.id), 'mean']], granularities=[numpy.timedelta64(1,
            'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_mix(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['rolling', 'sum', 2, ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 13, 0, 0), numpy.
            timedelta64(1, 'h'), 705), (datetime64(2014, 1, 1, 14, 0, 0),
            numpy.timedelta64(1, 'h'), 100), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 192)], values)

    def test_bool(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['gt', ['/', ['*', ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']],
            100], 1000], 10], granularities=[numpy.timedelta64(1, 'h')])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 1), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 14, 0, 0
            ), numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 1)], values)

    def test_unary_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['abs', ['metric', [str(self.metric.id),
            'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 4),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 0,
            0), numpy.timedelta64(1, 'h'), 9), (datetime64(2014, 1, 1, 13, 
            0, 0), numpy.timedelta64(1, 'h'), 2), (datetime64(2014, 1, 1, 
            14, 0, 0), numpy.timedelta64(1, 'h'), 4), (datetime64(2014, 1, 
            1, 15, 0, 0), numpy.timedelta64(1, 'h'), 4)]}}, values)

    def test_max_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['max', ['metric', str(self.metric.id),
            'mean'], ['*', ['metric', str(metric2.id), 'mean'], 2]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), -18), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 42), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 8), (datetime64(2014, 1, 1, 15, 
            0, 0), numpy.timedelta64(1, 'h'), 44)], values)

    def test_rateofchangesec_operator(self):
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 0), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 10), 450), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 5), 900), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 30), 1350)])
        self.trigger_processing([self.metric])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['rateofchangesec', [
            'metric', str(self.metric.id), 'mean']], granularities=[numpy.
            timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(5, 'm'), eq_nan), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 
            1.5), (datetime64(2014, 1, 1, 12, 10, 0), numpy.timedelta64(5,
            'm'), 1.5), (datetime64(2014, 1, 1, 12, 15, 0), numpy.
            timedelta64(5, 'm'), 1.5)]}}, values)

<mask token>


class TestAggregatedTimeseries(base.BaseTestCase):

    @staticmethod
    def _resample_and_merge(ts, agg_dict):
        """Helper method that mimics _compute_splits_operations workflow."""
        grouped = ts.group_serie(agg_dict['sampling'])
        existing = agg_dict.get('return')
        name = agg_dict.get('name')
        resource = None if name is None else mock.Mock(id=str(uuid.uuid4()))
        metric = mock.Mock(id=str(uuid.uuid4()), name=name)
        agg_dict['return'] = processor.MetricReference(metric, 'mean', resource
            ), carbonara.AggregatedTimeSerie.from_grouped_serie(grouped,
            carbonara.Aggregation(agg_dict['agg'], agg_dict['sampling'], None))
        if existing:
            existing[2].merge(agg_dict['return'][2])
            agg_dict['return'] = existing

    def test_aggregated_different_archive_no_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean', 'name': 'all'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean', 'name': 'all'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4)
            ], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 9, 1, 4), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        dtfrom = datetime64(2014, 1, 1, 11, 0, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, operations=['aggregate', 'mean', ['metric', ['all',
            'mean']]])

    def test_aggregated_different_archive_no_overlap2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 50, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = carbonara.AggregatedTimeSerie(carbonara.Aggregation('mean',
            numpy.timedelta64(60, 's'), None))
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        metric = mock.Mock(id=str(uuid.uuid4()))
        ref = processor.MetricReference(metric, 'mean')
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], (ref, tsc2)], operations=[
            'aggregate', 'mean', ['metric', tsc1['return'][0].lookup_key,
            ref.lookup_key]])

    def test_aggregated_different_archive_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 4),
            (datetime64(2014, 1, 1, 12, 1, 0), 3), (datetime64(2014, 1, 1, 
            12, 2, 0), 2), (datetime64(2014, 1, 1, 12, 3, 0), 4), (
            datetime64(2014, 1, 1, 12, 4, 0), 2), (datetime64(2014, 1, 1, 
            12, 5, 0), 3), (datetime64(2014, 1, 1, 12, 6, 0), 4), (
            datetime64(2014, 1, 1, 12, 7, 0), 10), (datetime64(2014, 1, 1, 
            12, 9, 0), 2)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 4), (datetime64(2014, 1, 1, 
            12, 3, 0), 4), (datetime64(2014, 1, 1, 12, 4, 0), 6), (
            datetime64(2014, 1, 1, 12, 5, 0), 3), (datetime64(2014, 1, 1, 
            12, 6, 0), 6), (datetime64(2014, 1, 1, 12, 9, 0), 2), (
            datetime64(2014, 1, 1, 12, 11, 0), 2), (datetime64(2014, 1, 1, 
            12, 12, 0), 2)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        dtfrom = datetime64(2014, 1, 1, 12, 0, 0)
        dtto = datetime64(2014, 1, 1, 12, 10, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, to_timestamp=dtto, operations=['aggregate', 'mean', [
            'metric', tsc1['return'][0].lookup_key, tsc2['return'][0].
            lookup_key]])
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto, operations=[
            'aggregate', 'mean', ['metric', tsc1['return'][0].lookup_key,
            tsc2['return'][0].lookup_key]], needed_percent_of_overlap=80.0)[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60, 's'), 3.0), (datetime64(2014, 1, 1, 12, 2, 0),
            numpy.timedelta64(60, 's'), 3.0), (datetime64(2014, 1, 1, 12, 3,
            0), numpy.timedelta64(60, 's'), 4.0), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60, 's'), 4.0), (datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(60, 's'), 3.0), (datetime64(
            2014, 1, 1, 12, 6, 0), numpy.timedelta64(60, 's'), 5.0), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60, 's'), 
            10.0), (datetime64(2014, 1, 1, 12, 9, 0), numpy.timedelta64(60,
            's'), 2.0)], list(output))

    def test_aggregated_different_archive_overlap_edge_missing1(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 5, 0), 2), (datetime64(2014, 1, 1, 12, 6, 0), 7), (
            datetime64(2014, 1, 1, 12, 7, 0), 5), (datetime64(2014, 1, 1, 
            12, 8, 0), 3)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4), (datetime64(2014, 1, 1, 
            12, 5, 0), 16), (datetime64(2014, 1, 1, 12, 6, 0), 12)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60, 's'), 33.0), (datetime64(2014, 1, 1, 12, 4, 0),
            numpy.timedelta64(60, 's'), 5.0), (datetime64(2014, 1, 1, 12, 5,
            0), numpy.timedelta64(60, 's'), 18.0), (datetime64(2014, 1, 1, 
            12, 6, 0), numpy.timedelta64(60, 's'), 19.0)], list(output))
    <mask token>

    def test_processor_with_random_holes_derived_boundaries(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1), (datetime64(2014, 1, 1, 
            12, 4, 0), 5), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60000000000, 'ns'), 5.5), (datetime64(2014, 1, 1, 
            12, 2, 0), numpy.timedelta64(60000000000, 'ns'), 7.0), (
            datetime64(2014, 1, 1, 12, 3, 0), numpy.timedelta64(60000000000,
            'ns'), 24.0), (datetime64(2014, 1, 1, 12, 4, 0), numpy.
            timedelta64(60000000000, 'ns'), 4.5)], list(output))

    def test_processor_derived_missing_boundary(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1), (datetime64(2014, 1, 1, 
            12, 4, 0), 5), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], from_timestamp=
            datetime64(2014, 1, 1, 12, 0, 0), needed_percent_of_overlap=50.0)[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), 6.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), 5.5), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), 7.0), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), 24.0), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), 4.5)], list(
            output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], to_timestamp=
            datetime64(2014, 1, 1, 12, 7, 0), needed_percent_of_overlap=50.0)[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 1, 0), numpy.
            timedelta64(60000000000, 'ns'), 5.5), (datetime64(2014, 1, 1, 
            12, 2, 0), numpy.timedelta64(60000000000, 'ns'), 7.0), (
            datetime64(2014, 1, 1, 12, 3, 0), numpy.timedelta64(60000000000,
            'ns'), 24.0), (datetime64(2014, 1, 1, 12, 4, 0), numpy.
            timedelta64(60000000000, 'ns'), 4.5), (datetime64(2014, 1, 1, 
            12, 6, 0), numpy.timedelta64(60000000000, 'ns'), 3.0)], list(
            output))
    <mask token>

    def test_aggregated_some_overlap_with_fill_null(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 5, 0), 2), (datetime64(2014, 1, 1, 12, 6, 0), 7), (
            datetime64(2014, 1, 1, 12, 7, 0), 5), (datetime64(2014, 1, 1, 
            12, 8, 0), 3)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4), (datetime64(2014, 1, 1, 
            12, 5, 0), 16), (datetime64(2014, 1, 1, 12, 6, 0), 12)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], fill='null')[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), 6.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), 2.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), 13.0), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), 16.5), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), 2.5), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(60000000000,
            'ns'), 9.0), (datetime64(2014, 1, 1, 12, 6, 0), numpy.
            timedelta64(60000000000, 'ns'), 9.5), (datetime64(2014, 1, 1, 
            12, 7, 0), numpy.timedelta64(60000000000, 'ns'), 5.0), (
            datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64(60000000000,
            'ns'), 3.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['-', ['metric'] + tsc1['return'][0].lookup_key, [
            'metric'] + tsc2['return'][0].lookup_key], fill='null')[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), eq_nan), (datetime64(2014, 1, 1,
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), eq_nan), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), eq_nan), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), -15.0), (datetime64(2014, 1, 1,
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), -3.0), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(60000000000,
            'ns'), -14.0), (datetime64(2014, 1, 1, 12, 6, 0), numpy.
            timedelta64(60000000000, 'ns'), -5.0), (datetime64(2014, 1, 1, 
            12, 7, 0), numpy.timedelta64(60000000000, 'ns'), eq_nan), (
            datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64(60000000000,
            'ns'), eq_nan)], list(output))

    def test_aggregate_no_points_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1), (datetime64(2014, 1, 1, 
            12, 7, 0), 5), (datetime64(2014, 1, 1, 12, 8, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            functools.partial(self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2), (datetime64(2014, 1, 1, 
            12, 2, 0), 13), (datetime64(2014, 1, 1, 12, 3, 0), 24), (
            datetime64(2014, 1, 1, 12, 4, 0), 4)], dtype=carbonara.
            TIMESERIES_ARRAY_DTYPE), before_truncate_callback=functools.
            partial(self._resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'mean', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]], fill=0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), 3.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), 1.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), 6.5), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), 16.5), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), 2.5), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60000000000,
            'ns'), 2.5), (datetime64(2014, 1, 1, 12, 8, 0), numpy.
            timedelta64(60000000000, 'ns'), 1.5)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['-', ['metric'] + tsc1['return'][0].lookup_key, [
            'metric'] + tsc2['return'][0].lookup_key], fill=0)['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(60000000000, 'ns'), -6.0), (datetime64(2014, 1, 1, 
            12, 1, 0), numpy.timedelta64(60000000000, 'ns'), -2.0), (
            datetime64(2014, 1, 1, 12, 2, 0), numpy.timedelta64(60000000000,
            'ns'), -13), (datetime64(2014, 1, 1, 12, 3, 0), numpy.
            timedelta64(60000000000, 'ns'), -15), (datetime64(2014, 1, 1, 
            12, 4, 0), numpy.timedelta64(60000000000, 'ns'), -3), (
            datetime64(2014, 1, 1, 12, 7, 0), numpy.timedelta64(60000000000,
            'ns'), 5), (datetime64(2014, 1, 1, 12, 8, 0), numpy.timedelta64
            (60000000000, 'ns'), 3)], list(output))

    def test_aggregated_nominal(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsc12 = {'sampling': numpy.timedelta64(300, 's'), 'size': 6, 'agg':
            'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc12['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'), 'size': 10, 'agg':
            'mean'}
        tsc22 = {'sampling': numpy.timedelta64(300, 's'), 'size': 6, 'agg':
            'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc22['sampling'])

        def ts1_update(ts):
            self._resample_and_merge(ts, tsc1)
            self._resample_and_merge(ts, tsc12)

        def ts2_update(ts):
            self._resample_and_merge(ts, tsc2)
            self._resample_and_merge(ts, tsc22)
        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4),
            (datetime64(2014, 1, 1, 11, 47, 34), 8), (datetime64(2014, 1, 1,
            11, 50, 54), 50), (datetime64(2014, 1, 1, 11, 54, 45), 4), (
            datetime64(2014, 1, 1, 11, 56, 49), 4), (datetime64(2014, 1, 1,
            11, 57, 22), 6), (datetime64(2014, 1, 1, 11, 58, 22), 5), (
            datetime64(2014, 1, 1, 12, 1, 4), 4), (datetime64(2014, 1, 1, 
            12, 1, 9), 7), (datetime64(2014, 1, 1, 12, 2, 1), 15), (
            datetime64(2014, 1, 1, 12, 2, 12), 1), (datetime64(2014, 1, 1, 
            12, 3, 0), 3), (datetime64(2014, 1, 1, 12, 4, 9), 7), (
            datetime64(2014, 1, 1, 12, 5, 1), 15), (datetime64(2014, 1, 1, 
            12, 5, 12), 1), (datetime64(2014, 1, 1, 12, 6, 0), 3)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            ts1_update)
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 6),
            (datetime64(2014, 1, 1, 11, 47, 34), 5), (datetime64(2014, 1, 1,
            11, 50, 54), 51), (datetime64(2014, 1, 1, 11, 54, 45), 5), (
            datetime64(2014, 1, 1, 11, 56, 49), 5), (datetime64(2014, 1, 1,
            11, 57, 22), 7), (datetime64(2014, 1, 1, 11, 58, 22), 5), (
            datetime64(2014, 1, 1, 12, 1, 4), 5), (datetime64(2014, 1, 1, 
            12, 1, 9), 8), (datetime64(2014, 1, 1, 12, 2, 1), 10), (
            datetime64(2014, 1, 1, 12, 2, 12), 2), (datetime64(2014, 1, 1, 
            12, 3, 0), 6), (datetime64(2014, 1, 1, 12, 4, 9), 4), (
            datetime64(2014, 1, 1, 12, 5, 1), 10), (datetime64(2014, 1, 1, 
            12, 5, 12), 1), (datetime64(2014, 1, 1, 12, 6, 0), 1)], dtype=
            carbonara.TIMESERIES_ARRAY_DTYPE), before_truncate_callback=
            ts2_update)
        output = processor.aggregated([tsc1['return'], tsc12['return'],
            tsc2['return'], tsc22['return']], operations=['aggregate',
            'mean', ['metric', tsc1['return'][0].lookup_key, tsc12['return'
            ][0].lookup_key, tsc2['return'][0].lookup_key, tsc22['return'][
            0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 11, 45), numpy.
            timedelta64(300, 's'), 5.75), (datetime64(2014, 1, 1, 11, 50),
            numpy.timedelta64(300, 's'), 27.5), (datetime64(2014, 1, 1, 11,
            55), numpy.timedelta64(300, 's'), 5.333333333333334), (
            datetime64(2014, 1, 1, 12, 0), numpy.timedelta64(300, 's'), 6.0
            ), (datetime64(2014, 1, 1, 12, 5), numpy.timedelta64(300, 's'),
            5.166666666666666), (numpy.datetime64(
            '2014-01-01T11:46:00.000000000'), numpy.timedelta64(60, 's'), 
            5.0), (numpy.datetime64('2014-01-01T11:47:00.000000000'), numpy
            .timedelta64(60, 's'), 6.5), (numpy.datetime64(
            '2014-01-01T11:50:00.000000000'), numpy.timedelta64(60, 's'), 
            50.5), (datetime64(2014, 1, 1, 11, 54), numpy.timedelta64(60,
            's'), 4.5), (datetime64(2014, 1, 1, 11, 56), numpy.timedelta64(
            60, 's'), 4.5), (datetime64(2014, 1, 1, 11, 57), numpy.
            timedelta64(60, 's'), 6.5), (datetime64(2014, 1, 1, 11, 58),
            numpy.timedelta64(60, 's'), 5.0), (datetime64(2014, 1, 1, 12, 1
            ), numpy.timedelta64(60, 's'), 6.0), (datetime64(2014, 1, 1, 12,
            2), numpy.timedelta64(60, 's'), 7.0), (datetime64(2014, 1, 1, 
            12, 3), numpy.timedelta64(60, 's'), 4.5), (datetime64(2014, 1, 
            1, 12, 4), numpy.timedelta64(60, 's'), 5.5), (datetime64(2014, 
            1, 1, 12, 5), numpy.timedelta64(60, 's'), 6.75), (datetime64(
            2014, 1, 1, 12, 6), numpy.timedelta64(60, 's'), 2.0)], list(output)
            )

    def test_aggregated_partial_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(1, 's'), 'size': 86400, 'agg':
            'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(1, 's'), 'size': 60, 'agg':
            'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])
        tsb1.set_values(numpy.array([(datetime64(2015, 12, 3, 13, 19, 15), 
            1), (datetime64(2015, 12, 3, 13, 20, 15), 1), (datetime64(2015,
            12, 3, 13, 21, 15), 1), (datetime64(2015, 12, 3, 13, 22, 15), 1
            )], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2015, 12, 3, 13, 21, 15), 
            10), (datetime64(2015, 12, 3, 13, 22, 15), 10), (datetime64(
            2015, 12, 3, 13, 23, 15), 10), (datetime64(2015, 12, 3, 13, 24,
            15), 10)], dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(self.
            _resample_and_merge, agg_dict=tsc2))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0)], list(output))
        dtfrom = datetime64(2015, 12, 3, 13, 17, 0)
        dtto = datetime64(2015, 12, 3, 13, 25, 0)
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto, operations=[
            'aggregate', 'sum', ['metric', tsc1['return'][0].lookup_key,
            tsc2['return'][0].lookup_key]], needed_percent_of_overlap=0)[
            'aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 19, 15), numpy.
            timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 20, 15),
            numpy.timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 
            21, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12,
            3, 13, 22, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(
            2015, 12, 3, 13, 23, 15), numpy.timedelta64(1, 's'), 10.0), (
            datetime64(2015, 12, 3, 13, 24, 15), numpy.timedelta64(1, 's'),
            10.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0)], list(output))
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], to_timestamp=dtto,
            operations=['aggregate', 'sum', ['metric', tsc1['return'][0].
            lookup_key, tsc2['return'][0].lookup_key]])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            aggregated, [tsc1['return'], tsc2['return']], from_timestamp=
            dtfrom, operations=['aggregate', 'sum', ['metric', tsc1[
            'return'][0].lookup_key, tsc2['return'][0].lookup_key]])
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, operations=['aggregate', 'sum', [
            'metric', tsc1['return'][0].lookup_key, tsc2['return'][0].
            lookup_key]], needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 19, 15), numpy.
            timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 20, 15),
            numpy.timedelta64(1, 's'), 1.0), (datetime64(2015, 12, 3, 13, 
            21, 15), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12,
            3, 13, 22, 15), numpy.timedelta64(1, 's'), 11.0)], list(output))
        output = processor.aggregated([tsc1['return'], tsc2['return']],
            to_timestamp=dtto, operations=['aggregate', 'sum', ['metric',
            tsc1['return'][0].lookup_key, tsc2['return'][0].lookup_key]],
            needed_percent_of_overlap=50.0)['aggregated']
        self.assertEqual([(datetime64(2015, 12, 3, 13, 21, 15), numpy.
            timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0), (datetime64(2015, 12, 3, 
            13, 23, 15), numpy.timedelta64(1, 's'), 10.0), (datetime64(2015,
            12, 3, 13, 24, 15), numpy.timedelta64(1, 's'), 10.0)], list(output)
            )


class CrossMetricAggregated(base.TestCase):

    def setUp(self):
        super(CrossMetricAggregated, self).setUp()
        self.metric, __ = self._create_metric()

    def test_get_measures_empty_metrics_no_overlap(self):
        references = [processor.MetricReference(indexer.Metric(uuid.uuid4(),
            self.archive_policies['low']), 'mean'), processor.
            MetricReference(indexer.Metric(uuid.uuid4(), self.
            archive_policies['low']), 'mean')]
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, references, operations=['aggregate',
            'mean', ['metric', ['whatever', 'mean'], ['everwhat', 'mean']]])

    def test_get_measures_empty_metric_needed_overlap_zero(self):
        m_id = str(self.metric.id)
        result = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], operations=['metric',
            m_id, 'mean'], needed_overlap=0)
        self.assertEqual({m_id: {'mean': []}}, result)

    def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(storage.AggregationDoesNotExist, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'last'), processor.MetricReference(metric2, 'last')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'last'], [metric2.id, 'last']]])

    def test_get_measures_unknown_granularity(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(12345456, 'ms')])

    def test_add_and_get_measures_different_archives(self):
        metric2 = indexer.Metric(uuid.uuid4(), self.archive_policies[
            'no_granularity_match'])
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 1), 69), incoming.Measure(datetime64(2014, 
            1, 1, 12, 7, 31), 42), incoming.Measure(datetime64(2014, 1, 1, 
            12, 9, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 12,
            45), 44)])
        self.assertRaises(exceptions.UnAggregableTimeseries, processor.
            get_measures, self.storage, [processor.MetricReference(self.
            metric, 'mean'), processor.MetricReference(metric2, 'mean')],
            operations=['aggregate', 'mean', ['metric', [str(self.metric.id
            ), 'mean'], [str(metric2.id), 'mean']]])

    def test_add_and_get_measures(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 7, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 39.0), (datetime64(2014, 1, 1,
            12, 5, 0), numpy.timedelta64(5, 'm'), 12.5), (datetime64(2014, 
            1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'max', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 39.75), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 39.75), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 69), (datetime64(2014, 1, 1, 
            12, 5, 0), numpy.timedelta64(5, 'm'), 23), (datetime64(2014, 1,
            1, 12, 10, 0), numpy.timedelta64(5, 'm'), 44)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 0))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12), numpy.timedelta64(1,
            'h'), 22.25), (datetime64(2014, 1, 1, 12, 10, 0), numpy.
            timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            to_timestamp=datetime64(2014, 1, 1, 12, 5, 0))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 
            0, 0), numpy.timedelta64(5, 'm'), 39.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 10), to_timestamp
            =datetime64(2014, 1, 1, 12, 10, 10))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12), numpy.timedelta64(1,
            'h'), 22.25), (datetime64(2014, 1, 1, 12, 10), numpy.
            timedelta64(5, 'm'), 24.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0), to_timestamp=
            datetime64(2014, 1, 1, 12, 0, 1))['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1), numpy.timedelta64(1, 'D'
            ), 22.25), (datetime64(2014, 1, 1, 12, 0, 0), numpy.timedelta64
            (1, 'h'), 22.25), (datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(5, 'm'), 39.0)], values)
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0), to_timestamp=
            datetime64(2014, 1, 1, 12, 0, 1), granularities=[numpy.
            timedelta64(5, 'm')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(5, 'm'), 39.0)], values)

    def test_add_and_get_measures_with_holes(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 7, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 8), incoming.Measure(
            datetime64(2014, 1, 1, 12, 9, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 12, 45), 42)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 7, 31), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            9, 31), 6), incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10),
            2)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], operations=['aggregate', 'mean', ['metric',
            [str(self.metric.id), 'mean'], [str(metric2.id), 'mean']]])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 18.875), (datetime64(2014, 1, 1, 12, 0, 0),
            numpy.timedelta64(1, 'h'), 18.875), (datetime64(2014, 1, 1, 12,
            0, 0), numpy.timedelta64(5, 'm'), 39.0), (datetime64(2014, 1, 1,
            12, 5, 0), numpy.timedelta64(5, 'm'), 11.0), (datetime64(2014, 
            1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 22.0)], values)

    def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['resample', 'mean', numpy.timedelta64(1,
            'D'), ['metric', [str(self.metric.id), 'mean'], [str(metric2.id
            ), 'mean']]], granularities=[numpy.timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 39.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 4.75)]}}, values)

    def test_resample_minus_2_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]], 2], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), 37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), 2.75)]}}, values)

    def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['-', 2, ['resample', 'mean', numpy.
            timedelta64(1, 'D'), ['metric', [str(self.metric.id), 'mean'],
            [str(metric2.id), 'mean']]]], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 0, 0, 0), numpy.timedelta64(1, 'D'), -37.75)]}, str(
            metric2.id): {'mean': [(datetime64(2014, 1, 1, 0, 0, 0), numpy.
            timedelta64(1, 'D'), -2.75)]}}, values)

    def test_rolling(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 12, 5, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 12, 
            10, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 12, 15, 10
            ), 4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['/', ['rolling', 'sum', 2, ['metric', [str
            (self.metric.id), 'mean'], [str(metric2.id), 'mean']]], 2],
            granularities=[numpy.timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 55.5), (datetime64(
            2014, 1, 1, 12, 10, 0), numpy.timedelta64(5, 'm'), 23), (
            datetime64(2014, 1, 1, 12, 15, 0), numpy.timedelta64(5, 'm'), 
            24)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 5,
            0), numpy.timedelta64(5, 'm'), 5.5), (datetime64(2014, 1, 1, 12,
            10, 0), numpy.timedelta64(5, 'm'), 3), (datetime64(2014, 1, 1, 
            12, 15, 0), numpy.timedelta64(5, 'm'), 4)]}}, values)

    def test_binary_operator_with_two_references(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['*', ['metric', str(self.metric.id),
            'mean'], ['metric', str(metric2.id), 'mean']], granularities=[
            numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 621), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 84), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 16), (datetime64(2014, 1, 1, 15,
            0, 0), numpy.timedelta64(1, 'h'), 176)], values)

    def test_binary_operator_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', ['metric', str(
            self.metric.id), 'mean'], 2], granularities=[numpy.timedelta64(
            1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_ternary_operator_clip_min_max_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 5, 60], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 60), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 5),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}}, values)

    def test_ternary_operator_clip_min_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['clip', ['metric', str(
            self.metric.id), 'mean'], 50], granularities=[numpy.timedelta64
            (1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 50), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 50
            ), (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'),
            50)]}}, values)

    def test_binary_operator_ts_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.trigger_processing()
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['*', 2, ['metric', str(
            self.metric.id), 'mean']], granularities=[numpy.timedelta64(1,
            'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 138), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 84), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 8),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            88)]}}, values)

    def test_mix(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['rolling', 'sum', 2, ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 13, 0, 0), numpy.
            timedelta64(1, 'h'), 705), (datetime64(2014, 1, 1, 14, 0, 0),
            numpy.timedelta64(1, 'h'), 100), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 192)], values)

    def test_bool(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), 4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), 9), incoming.Measure(datetime64(2014, 1,
            1, 13, 1, 41), 2), incoming.Measure(datetime64(2014, 1, 1, 14, 
            2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4)]
            )
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['gt', ['/', ['*', ['*', ['metric', str(
            self.metric.id), 'mean'], ['metric', str(metric2.id), 'mean']],
            100], 1000], 10], granularities=[numpy.timedelta64(1, 'h')])[
            'aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), 1), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 14, 0, 0
            ), numpy.timedelta64(1, 'h'), 0), (datetime64(2014, 1, 1, 15, 0,
            0), numpy.timedelta64(1, 'h'), 1)], values)

    def test_unary_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['abs', ['metric', [str(self.metric.id),
            'mean'], [str(metric2.id), 'mean']]], granularities=[numpy.
            timedelta64(1, 'h')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(1, 'h'), 69), (datetime64(
            2014, 1, 1, 13, 0, 0), numpy.timedelta64(1, 'h'), 42), (
            datetime64(2014, 1, 1, 14, 0, 0), numpy.timedelta64(1, 'h'), 4),
            (datetime64(2014, 1, 1, 15, 0, 0), numpy.timedelta64(1, 'h'), 
            44)]}, str(metric2.id): {'mean': [(datetime64(2014, 1, 1, 12, 0,
            0), numpy.timedelta64(1, 'h'), 9), (datetime64(2014, 1, 1, 13, 
            0, 0), numpy.timedelta64(1, 'h'), 2), (datetime64(2014, 1, 1, 
            14, 0, 0), numpy.timedelta64(1, 'h'), 4), (datetime64(2014, 1, 
            1, 15, 0, 0), numpy.timedelta64(1, 'h'), 4)]}}, values)

    def test_max_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), -69), incoming.Measure(
            datetime64(2014, 1, 1, 13, 1, 31), 42), incoming.Measure(
            datetime64(2014, 1, 1, 14, 2, 31), -4), incoming.Measure(
            datetime64(2014, 1, 1, 15, 3, 45), 44)])
        self.incoming.add_measures(metric2.id, [incoming.Measure(datetime64
            (2014, 1, 1, 12, 0, 5), -9), incoming.Measure(datetime64(2014, 
            1, 1, 13, 1, 41), -2), incoming.Measure(datetime64(2014, 1, 1, 
            14, 2, 31), 4), incoming.Measure(datetime64(2014, 1, 1, 15, 3, 
            10), -4)])
        self.trigger_processing([self.metric, metric2])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean'), processor.MetricReference
            (metric2, 'mean')], ['max', ['metric', str(self.metric.id),
            'mean'], ['*', ['metric', str(metric2.id), 'mean'], 2]],
            granularities=[numpy.timedelta64(1, 'h')])['aggregated']
        self.assertEqual([(datetime64(2014, 1, 1, 12, 0, 0), numpy.
            timedelta64(1, 'h'), -18), (datetime64(2014, 1, 1, 13, 0, 0),
            numpy.timedelta64(1, 'h'), 42), (datetime64(2014, 1, 1, 14, 0, 
            0), numpy.timedelta64(1, 'h'), 8), (datetime64(2014, 1, 1, 15, 
            0, 0), numpy.timedelta64(1, 'h'), 44)], values)

    def test_rateofchangesec_operator(self):
        self.incoming.add_measures(self.metric.id, [incoming.Measure(
            datetime64(2014, 1, 1, 12, 0, 1), 0), incoming.Measure(
            datetime64(2014, 1, 1, 12, 5, 10), 450), incoming.Measure(
            datetime64(2014, 1, 1, 12, 10, 5), 900), incoming.Measure(
            datetime64(2014, 1, 1, 12, 15, 30), 1350)])
        self.trigger_processing([self.metric])
        values = processor.get_measures(self.storage, [processor.
            MetricReference(self.metric, 'mean')], ['rateofchangesec', [
            'metric', str(self.metric.id), 'mean']], granularities=[numpy.
            timedelta64(5, 'm')])
        self.assertEqual({str(self.metric.id): {'mean': [(datetime64(2014, 
            1, 1, 12, 0, 0), numpy.timedelta64(5, 'm'), eq_nan), (
            datetime64(2014, 1, 1, 12, 5, 0), numpy.timedelta64(5, 'm'), 
            1.5), (datetime64(2014, 1, 1, 12, 10, 0), numpy.timedelta64(5,
            'm'), 1.5), (datetime64(2014, 1, 1, 12, 15, 0), numpy.
            timedelta64(5, 'm'), 1.5)]}}, values)

# -*- encoding: utf-8 -*-
#
# Copyright © 2014-2016 eNovance
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import functools
import uuid

import numpy
from unittest import mock

from gnocchi import carbonara
from gnocchi import incoming
from gnocchi import indexer
from gnocchi.rest.aggregates import api
from gnocchi.rest.aggregates import exceptions
from gnocchi.rest.aggregates import processor
from gnocchi import storage
from gnocchi.tests import base


class EqNan(object):
    def __eq__(self, other):
        return numpy.isnan(other)


eq_nan = EqNan()


def datetime64(*args):
    return numpy.datetime64(datetime.datetime(*args))


class TestGrouper(base.BaseTestCase):
    def setUp(self):
        super(TestGrouper, self).setUp()
        self.body = {"operations": "mean cpu.util"}
        self.references = self.body['operations']

        self.attribute_filters = {"attribute1": "value_attribute_1"}

        self.group_by = ["attribute1", "attribute2", "attribute3"]
        self.sorts = self.group_by

        self.start = datetime.datetime.now()
        self.end = self.start + datetime.timedelta(hours=1)

        self.granularity = 3600

        self.need_overlap = None
        self.fill = None

        self.details = True

        self.grouper_test = api.Grouper(self.group_by, self.start, self.end,
                                        self.body, self.sorts,
                                        self.attribute_filters,
                                        self.references, self.granularity,
                                        self.need_overlap, self.fill,
                                        self.details)

        self.test_resource_1 = {
            'uuid': '30b51786-944f-427b-85df-ce5f462e58a4',
            'revision_start': self.start - datetime.timedelta(hours=10),
            'revision_end': self.start - datetime.timedelta(hours=9),
            "attribute1": "a", "attribute2": "a", "attribute3": "a"
        }
        self.test_resource_2 = {
            'uuid': '17002c42-2f30-45fb-b1f1-eb33f7345e89',
            'revision_start': self.start - datetime.timedelta(hours=7),
            'revision_end': self.start - datetime.timedelta(hours=5),
            "attribute1": None, "attribute2": "b", "attribute3": "b"
        }
        self.test_resource_3 = {
            'uuid': '8f6be3bb-d5ee-4d01-986b-c034386637b0',
            'revision_start': self.start - datetime.timedelta(hours=2),
            'revision_end': self.start - datetime.timedelta(hours=1),
            "attribute1": "c", "attribute2": None, "attribute3": None
        }
        self.test_resources = [self.test_resource_1, self.test_resource_2,
                               self.test_resource_3]

    def test_create_history_period_filter_no_start(self):
        self.grouper_test.start = None
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual(self.attribute_filters, return_of_method)

    def test_create_history_period_filter_no_end(self):
        self.grouper_test.end = None
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual(self.attribute_filters, return_of_method)

    def test_create_history_period_filter_no_attribute_filter(self):
        self.grouper_test.attr_filter = None
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual({"and": [
            {"<": {'revision_start': self.end}},
            {"or": [
                {">=": {'revision_end': self.start}},
                {"=": {'revision_end': None}}]}]},
            return_of_method)

    def test_create_history_period_filter(self):
        return_of_method = self.grouper_test.create_history_period_filter()

        self.assertEqual({"and": [{"and": [
            {"<": {'revision_start': self.end}},
            {"or": [
                {">=": {'revision_end': self.start}},
                {"=": {'revision_end': None}}]}]}, self.attribute_filters]},
            return_of_method)

    def test_group_resources_with_all_attributes(self):
        response_of_method = self.grouper_test.group(self.test_resources)

        self.assertEqual("17002c42-2f30-45fb-b1f1-eb33f7345e89",
                         response_of_method[0].resources[0].get('uuid'))
        self.assertEqual("30b51786-944f-427b-85df-ce5f462e58a4",
                         response_of_method[1].resources[0].get('uuid'))
        self.assertEqual("8f6be3bb-d5ee-4d01-986b-c034386637b0",
                         response_of_method[2].resources[0].get('uuid'))


class TestAggregatedTimeseries(base.BaseTestCase):
    @staticmethod
    def _resample_and_merge(ts, agg_dict):
        """Helper method that mimics _compute_splits_operations workflow."""
        grouped = ts.group_serie(agg_dict['sampling'])
        existing = agg_dict.get('return')
        name = agg_dict.get("name")
        resource = None if name is None else mock.Mock(id=str(uuid.uuid4()))
        metric = mock.Mock(id=str(uuid.uuid4()), name=name)
        agg_dict['return'] = (
            processor.MetricReference(metric, "mean", resource),
            carbonara.AggregatedTimeSerie.from_grouped_serie(
                grouped,
                carbonara.Aggregation(agg_dict['agg'],
                                      agg_dict['sampling'],
                                      None)))
        if existing:
            existing[2].merge(agg_dict['return'][2])
            agg_dict['return'] = existing

    def test_aggregated_different_archive_no_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean', "name": "all"}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean', "name": "all"}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 46, 4), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))
        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 9, 1, 4), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc2))

        dtfrom = datetime64(2014, 1, 1, 11, 0, 0)
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          operations=["aggregate", "mean", [
                              "metric", ["all", "mean"]]])

    def test_aggregated_different_archive_no_overlap2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 50, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = carbonara.AggregatedTimeSerie(
            carbonara.Aggregation('mean', numpy.timedelta64(60, 's'), None))

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))
        metric = mock.Mock(id=str(uuid.uuid4()))
        ref = processor.MetricReference(metric, "mean")
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], (ref, tsc2)],
                          operations=["aggregate", "mean",
                                      ["metric", tsc1['return'][0].lookup_key,
                                       ref.lookup_key]])

    def test_aggregated_different_archive_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        # NOTE(sileht): minute 8 is missing in both and
        # minute 7 in tsc2 too, but it looks like we have
        # enough point to do the aggregation
        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 0, 0), 4),
            (datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 2),
            (datetime64(2014, 1, 1, 12, 3, 0), 4),
            (datetime64(2014, 1, 1, 12, 4, 0), 2),
            (datetime64(2014, 1, 1, 12, 5, 0), 3),
            (datetime64(2014, 1, 1, 12, 6, 0), 4),
            (datetime64(2014, 1, 1, 12, 7, 0), 10),
            (datetime64(2014, 1, 1, 12, 9, 0), 2)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 1, 0), 3),
            (datetime64(2014, 1, 1, 12, 2, 0), 4),
            (datetime64(2014, 1, 1, 12, 3, 0), 4),
            (datetime64(2014, 1, 1, 12, 4, 0), 6),
            (datetime64(2014, 1, 1, 12, 5, 0), 3),
            (datetime64(2014, 1, 1, 12, 6, 0), 6),
            (datetime64(2014, 1, 1, 12, 9, 0), 2),
            (datetime64(2014, 1, 1, 12, 11, 0), 2),
            (datetime64(2014, 1, 1, 12, 12, 0), 2)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        dtfrom = datetime64(2014, 1, 1, 12, 0, 0)
        dtto = datetime64(2014, 1, 1, 12, 10, 0)

        # By default we require 100% of point that overlap
        # so that fail
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          to_timestamp=dtto,
                          operations=["aggregate", "mean", [
                              "metric",
                              tsc1['return'][0].lookup_key,
                              tsc2['return'][0].lookup_key,
                          ]])

        # Retry with 80% and it works
        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto,
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key,
            ]],
            needed_percent_of_overlap=80.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60, 's'), 3.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60, 's'), 3.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60, 's'), 4.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60, 's'), 4.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60, 's'), 3.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60, 's'), 5.0),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60, 's'), 10.0),
            (datetime64(2014, 1, 1, 12, 9, 0),
             numpy.timedelta64(60, 's'), 2.0),
        ], list(output))

    def test_aggregated_different_archive_overlap_edge_missing1(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 5, 0), 2),
            (datetime64(2014, 1, 1, 12, 6, 0), 7),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4),
            (datetime64(2014, 1, 1, 12, 5, 0), 16),
            (datetime64(2014, 1, 1, 12, 6, 0), 12)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        # By default we require 100% of point that overlap
        # but we allow that the last datapoint is missing
        # of the precisest granularity
        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60, 's'), 33.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60, 's'), 5.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60, 's'), 18.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60, 's'), 19.0),
        ], list(output))

    def test_aggregated_different_archive_overlap_edge_missing2(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([(datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([(datetime64(2014, 1, 1, 11, 0, 0), 4),
                                     (datetime64(2014, 1, 1, 12, 3, 0), 4)],
                                    dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
                        before_truncate_callback=functools.partial(
                            self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(
                2014, 1, 1, 12, 3, 0
            ), numpy.timedelta64(60000000000, 'ns'), 4.0),
        ], list(output))

    def test_processor_with_random_holes_derived_boundaries(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1),
            (datetime64(2014, 1, 1, 12, 4, 0), 5),
            (datetime64(2014, 1, 1, 12, 6, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=50.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.5),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 7.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 24.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 4.5),
        ], list(output))

    def test_processor_derived_missing_boundary(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 1, 0), 9),
            (datetime64(2014, 1, 1, 12, 2, 0), 1),
            (datetime64(2014, 1, 1, 12, 4, 0), 5),
            (datetime64(2014, 1, 1, 12, 6, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0),
            needed_percent_of_overlap=50.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.5),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 7.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 24.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 4.5),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key,
            ]],
            to_timestamp=datetime64(2014, 1, 1, 12, 7, 0),
            needed_percent_of_overlap=50.0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.5),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 7.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 24.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 4.5),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
        ], list(output))

    def test_aggregated_some_overlap_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', "name": "foo"}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', "name": "bar"}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 5, 0), 2),
            (datetime64(2014, 1, 1, 12, 6, 0), 7),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4),
            (datetime64(2014, 1, 1, 12, 5, 0), 16),
            (datetime64(2014, 1, 1, 12, 6, 0), 12)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.5),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 16.5),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.5),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["-", ["metric"] + tsc1['return'][0].lookup_key,
                        ["metric"] + tsc2['return'][0].lookup_key
                        ], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), -6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), -2.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), -13),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), -15),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), -3),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), -14),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), -5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 3),
        ], list(output))

    def test_aggregated_some_overlap_with_fill_null(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 5, 0), 2),
            (datetime64(2014, 1, 1, 12, 6, 0), 7),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4),
            (datetime64(2014, 1, 1, 12, 5, 0), 16),
            (datetime64(2014, 1, 1, 12, 6, 0), 12)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], fill='null')["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 13.0),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 16.5),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), 9.5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 5.0),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["-", ["metric"] + tsc1['return'][0].lookup_key,
                        ["metric"] + tsc2['return'][0].lookup_key
                        ], fill='null')["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), -15.0),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), -3.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(60000000000, 'ns'), -14.0),
            (datetime64(2014, 1, 1, 12, 6, 0),
             numpy.timedelta64(60000000000, 'ns'), -5.0),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), eq_nan),
        ], list(output))

    def test_aggregate_no_points_with_fill_zero(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 3, 0), 9),
            (datetime64(2014, 1, 1, 12, 4, 0), 1),
            (datetime64(2014, 1, 1, 12, 7, 0), 5),
            (datetime64(2014, 1, 1, 12, 8, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 12, 0, 0), 6),
            (datetime64(2014, 1, 1, 12, 1, 0), 2),
            (datetime64(2014, 1, 1, 12, 2, 0), 13),
            (datetime64(2014, 1, 1, 12, 3, 0), 24),
            (datetime64(2014, 1, 1, 12, 4, 0), 4)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), 3.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), 6.5),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), 16.5),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 2.5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 1.5),
        ], list(output))

        output = processor.aggregated([
            tsc1['return'], tsc2['return']],
            operations=["-", ["metric"] + tsc1['return'][0].lookup_key,
                        ["metric"] + tsc2['return'][0].lookup_key
                        ], fill=0)["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(60000000000, 'ns'), -6.0),
            (datetime64(2014, 1, 1, 12, 1, 0),
             numpy.timedelta64(60000000000, 'ns'), -2.0),
            (datetime64(2014, 1, 1, 12, 2, 0),
             numpy.timedelta64(60000000000, 'ns'), -13),
            (datetime64(2014, 1, 1, 12, 3, 0),
             numpy.timedelta64(60000000000, 'ns'), -15),
            (datetime64(2014, 1, 1, 12, 4, 0),
             numpy.timedelta64(60000000000, 'ns'), -3),
            (datetime64(2014, 1, 1, 12, 7, 0),
             numpy.timedelta64(60000000000, 'ns'), 5),
            (datetime64(2014, 1, 1, 12, 8, 0),
             numpy.timedelta64(60000000000, 'ns'), 3),
        ], list(output))

    def test_aggregated_nominal(self):
        tsc1 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsc12 = {'sampling': numpy.timedelta64(300, 's'),
                 'size': 6, 'agg': 'mean'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc12['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(60, 's'),
                'size': 10, 'agg': 'mean'}
        tsc22 = {'sampling': numpy.timedelta64(300, 's'),
                 'size': 6, 'agg': 'mean'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc22['sampling'])

        def ts1_update(ts):
            self._resample_and_merge(ts, tsc1)
            self._resample_and_merge(ts, tsc12)

        def ts2_update(ts):
            self._resample_and_merge(ts, tsc2)
            self._resample_and_merge(ts, tsc22)

        tsb1.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 4), 4),
            (datetime64(2014, 1, 1, 11, 47, 34), 8),
            (datetime64(2014, 1, 1, 11, 50, 54), 50),
            (datetime64(2014, 1, 1, 11, 54, 45), 4),
            (datetime64(2014, 1, 1, 11, 56, 49), 4),
            (datetime64(2014, 1, 1, 11, 57, 22), 6),
            (datetime64(2014, 1, 1, 11, 58, 22), 5),
            (datetime64(2014, 1, 1, 12, 1, 4), 4),
            (datetime64(2014, 1, 1, 12, 1, 9), 7),
            (datetime64(2014, 1, 1, 12, 2, 1), 15),
            (datetime64(2014, 1, 1, 12, 2, 12), 1),
            (datetime64(2014, 1, 1, 12, 3, 0), 3),
            (datetime64(2014, 1, 1, 12, 4, 9), 7),
            (datetime64(2014, 1, 1, 12, 5, 1), 15),
            (datetime64(2014, 1, 1, 12, 5, 12), 1),
            (datetime64(2014, 1, 1, 12, 6, 0), 3)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=ts1_update)

        tsb2.set_values(numpy.array([
            (datetime64(2014, 1, 1, 11, 46, 4), 6),
            (datetime64(2014, 1, 1, 11, 47, 34), 5),
            (datetime64(2014, 1, 1, 11, 50, 54), 51),
            (datetime64(2014, 1, 1, 11, 54, 45), 5),
            (datetime64(2014, 1, 1, 11, 56, 49), 5),
            (datetime64(2014, 1, 1, 11, 57, 22), 7),
            (datetime64(2014, 1, 1, 11, 58, 22), 5),
            (datetime64(2014, 1, 1, 12, 1, 4), 5),
            (datetime64(2014, 1, 1, 12, 1, 9), 8),
            (datetime64(2014, 1, 1, 12, 2, 1), 10),
            (datetime64(2014, 1, 1, 12, 2, 12), 2),
            (datetime64(2014, 1, 1, 12, 3, 0), 6),
            (datetime64(2014, 1, 1, 12, 4, 9), 4),
            (datetime64(2014, 1, 1, 12, 5, 1), 10),
            (datetime64(2014, 1, 1, 12, 5, 12), 1),
            (datetime64(2014, 1, 1, 12, 6, 0), 1)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=ts2_update)
        output = processor.aggregated(
            [tsc1['return'], tsc12['return'], tsc2['return'], tsc22['return']],
            operations=["aggregate", "mean", [
                "metric",
                tsc1['return'][0].lookup_key, tsc12['return'][0].lookup_key,
                tsc2['return'][0].lookup_key, tsc22['return'][0].lookup_key
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 11, 45),
             numpy.timedelta64(300, 's'), 5.75),
            (datetime64(2014, 1, 1, 11, 50),
             numpy.timedelta64(300, 's'), 27.5),
            (datetime64(2014, 1, 1, 11, 55),
             numpy.timedelta64(300, 's'), 5.3333333333333339),
            (datetime64(2014, 1, 1, 12, 0),
             numpy.timedelta64(300, 's'), 6.0),
            (datetime64(2014, 1, 1, 12, 5),
             numpy.timedelta64(300, 's'), 5.1666666666666661),
            (numpy.datetime64('2014-01-01T11:46:00.000000000'),
             numpy.timedelta64(60, 's'),
             5.0),
            (numpy.datetime64('2014-01-01T11:47:00.000000000'),
             numpy.timedelta64(60, 's'),
             6.5),
            (numpy.datetime64('2014-01-01T11:50:00.000000000'),
             numpy.timedelta64(60, 's'),
             50.5),
            (datetime64(2014, 1, 1, 11, 54),
             numpy.timedelta64(60, 's'), 4.5),
            (datetime64(2014, 1, 1, 11, 56),
             numpy.timedelta64(60, 's'), 4.5),
            (datetime64(2014, 1, 1, 11, 57),
             numpy.timedelta64(60, 's'), 6.5),
            (datetime64(2014, 1, 1, 11, 58),
             numpy.timedelta64(60, 's'), 5.0),
            (datetime64(2014, 1, 1, 12, 1),
             numpy.timedelta64(60, 's'), 6.0),
            (datetime64(2014, 1, 1, 12, 2),
             numpy.timedelta64(60, 's'), 7.0),
            (datetime64(2014, 1, 1, 12, 3),
             numpy.timedelta64(60, 's'), 4.5),
            (datetime64(2014, 1, 1, 12, 4),
             numpy.timedelta64(60, 's'), 5.5),
            (datetime64(2014, 1, 1, 12, 5),
             numpy.timedelta64(60, 's'), 6.75),
            (datetime64(2014, 1, 1, 12, 6),
             numpy.timedelta64(60, 's'), 2.0),
        ], list(output))

    def test_aggregated_partial_overlap(self):
        tsc1 = {'sampling': numpy.timedelta64(1, 's'),
                'size': 86400, 'agg': 'mean', 'name': 'foo'}
        tsb1 = carbonara.BoundTimeSerie(block_size=tsc1['sampling'])
        tsc2 = {'sampling': numpy.timedelta64(1, 's'),
                'size': 60, 'agg': 'mean', 'name': 'bar'}
        tsb2 = carbonara.BoundTimeSerie(block_size=tsc2['sampling'])

        tsb1.set_values(numpy.array([
            (datetime64(2015, 12, 3, 13, 19, 15), 1),
            (datetime64(2015, 12, 3, 13, 20, 15), 1),
            (datetime64(2015, 12, 3, 13, 21, 15), 1),
            (datetime64(2015, 12, 3, 13, 22, 15), 1)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc1))

        tsb2.set_values(numpy.array([
            (datetime64(2015, 12, 3, 13, 21, 15), 10),
            (datetime64(2015, 12, 3, 13, 22, 15), 10),
            (datetime64(2015, 12, 3, 13, 23, 15), 10),
            (datetime64(2015, 12, 3, 13, 24, 15), 10)],
            dtype=carbonara.TIMESERIES_ARRAY_DTYPE),
            before_truncate_callback=functools.partial(
                self._resample_and_merge, agg_dict=tsc2))

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]])["aggregated"]

        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
        ], list(output))

        dtfrom = datetime64(2015, 12, 3, 13, 17, 0)
        dtto = datetime64(2015, 12, 3, 13, 25, 0)

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            from_timestamp=dtfrom, to_timestamp=dtto,
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 19, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 20, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 23, 15
            ), numpy.timedelta64(1, 's'), 10.0),
            (datetime64(
                2015, 12, 3, 13, 24, 15
            ), numpy.timedelta64(1, 's'), 10.0),
        ], list(output))

        # Check boundaries are set when overlap=0
        output = processor.aggregated(
            [tsc1['return'], tsc2['return']],
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
        ], list(output))

        # By default we require 100% of point that overlap
        # so that fail if from or to is set
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          to_timestamp=dtto,
                          operations=["aggregate", "sum", [
                              "metric",
                              tsc1['return'][0].lookup_key,
                              tsc2['return'][0].lookup_key
                          ]])
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.aggregated,
                          [tsc1['return'], tsc2['return']],
                          from_timestamp=dtfrom,
                          operations=["aggregate", "sum", [
                              "metric",
                              tsc1['return'][0].lookup_key,
                              tsc2['return'][0].lookup_key
                          ]])
        # Retry with 50% and it works
        output = processor.aggregated(
            [tsc1['return'], tsc2['return']], from_timestamp=dtfrom,
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=50.0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 19, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 20, 15
            ), numpy.timedelta64(1, 's'), 1.0),
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
        ], list(output))

        output = processor.aggregated(
            [tsc1['return'], tsc2['return']], to_timestamp=dtto,
            operations=["aggregate", "sum", [
                "metric",
                tsc1['return'][0].lookup_key,
                tsc2['return'][0].lookup_key
            ]], needed_percent_of_overlap=50.0)["aggregated"]
        self.assertEqual([
            (datetime64(
                2015, 12, 3, 13, 21, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 22, 15
            ), numpy.timedelta64(1, 's'), 11.0),
            (datetime64(
                2015, 12, 3, 13, 23, 15
            ), numpy.timedelta64(1, 's'), 10.0),
            (datetime64(
                2015, 12, 3, 13, 24, 15
            ), numpy.timedelta64(1, 's'), 10.0),
        ], list(output))


class CrossMetricAggregated(base.TestCase):
    def setUp(self):
        super(CrossMetricAggregated, self).setUp()
        # A lot of tests wants a metric, create one
        self.metric, __ = self._create_metric()

    def test_get_measures_empty_metrics_no_overlap(self):
        references = [
            processor.MetricReference(indexer.Metric(
                uuid.uuid4(), self.archive_policies['low']), 'mean'),
            processor.MetricReference(indexer.Metric(
                uuid.uuid4(), self.archive_policies['low']), 'mean'),
        ]
        self.assertRaises(
            exceptions.UnAggregableTimeseries,
            processor.get_measures, self.storage, references,
            operations=["aggregate", "mean", [
                "metric", ["whatever", "mean"], ["everwhat", "mean"],
            ]])

    def test_get_measures_empty_metric_needed_overlap_zero(self):
        m_id = str(self.metric.id)
        result = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            operations=["metric", m_id, "mean"], needed_overlap=0)
        self.assertEqual({m_id: {"mean": []}}, result)

    def test_get_measures_unknown_aggregation(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.assertRaises(storage.AggregationDoesNotExist,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, 'last'),
                           processor.MetricReference(metric2, 'last')],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "last"],
                              [(metric2.id), "last"],
                          ]])

    def test_get_measures_unknown_granularity(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['low'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, "mean"),
                           processor.MetricReference(metric2, "mean")],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "mean"],
                              [str(metric2.id), "mean"],
                          ]],
                          granularities=[numpy.timedelta64(12345456, 'ms')])

    def test_add_and_get_measures_different_archives(self):
        metric2 = indexer.Metric(uuid.uuid4(),
                                 self.archive_policies['no_granularity_match'])
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])

        self.assertRaises(exceptions.UnAggregableTimeseries,
                          processor.get_measures,
                          self.storage,
                          [processor.MetricReference(self.metric, "mean"),
                           processor.MetricReference(metric2, "mean")],
                          operations=["aggregate", "mean", [
                              "metric",
                              [str(self.metric.id), "mean"],
                              [str(metric2.id), "mean"],
                          ]])

    def test_add_and_get_measures(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 12.5),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 24.0)
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "max", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 39.75),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 39.75),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 69),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 23),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 44)
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 0))["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 24.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            to_timestamp=datetime64(2014, 1, 1, 12, 5, 0))["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 10, 10),
            to_timestamp=datetime64(2014, 1, 1, 12, 10, 10))["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 10),
             numpy.timedelta64(5, 'm'), 24.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0),
            to_timestamp=datetime64(2014, 1, 1, 12, 0, 1))["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1),
             numpy.timedelta64(1, 'D'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 22.25),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
        ], values)

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]],
            from_timestamp=datetime64(2014, 1, 1, 12, 0, 0),
            to_timestamp=datetime64(2014, 1, 1, 12, 0, 1),
            granularities=[numpy.timedelta64(5, 'm')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
        ], values)

    def test_add_and_get_measures_with_holes(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 31), 8),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 12, 45), 42),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 7, 31), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 9, 31), 6),
            incoming.Measure(datetime64(2014, 1, 1, 12, 13, 10), 2),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, 'mean'),
             processor.MetricReference(metric2, 'mean')],
            operations=["aggregate", "mean", [
                "metric",
                [str(self.metric.id), "mean"],
                [str(metric2.id), "mean"],
            ]])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 0, 0, 0),
             numpy.timedelta64(1, 'D'), 18.875),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 18.875),
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(5, 'm'), 39.0),
            (datetime64(2014, 1, 1, 12, 5, 0),
             numpy.timedelta64(5, 'm'), 11.0),
            (datetime64(2014, 1, 1, 12, 10, 0),
             numpy.timedelta64(5, 'm'), 22.0)
        ], values)

    def test_resample(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["resample", "mean", numpy.timedelta64(1, 'D'),
             ["metric",
              [str(self.metric.id), "mean"],
              [str(metric2.id), "mean"]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 39.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 4.75)]
            }
        }, values)

    def test_resample_minus_2_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["-", ["resample", "mean", numpy.timedelta64(1, 'D'),
                   ["metric",
                    [str(self.metric.id), "mean"],
                    [str(metric2.id), "mean"]]], 2],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 37.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), 2.75)]
            }
        }, values)

    def test_resample_minus_2_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["-",
             2,
             ["resample", "mean", numpy.timedelta64(1, 'D'),
              ["metric",
               [str(self.metric.id), "mean"],
               [str(metric2.id), "mean"]]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), -37.75)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 0, 0, 0),
                          numpy.timedelta64(1, 'D'), -2.75)]
            }
        }, values)

    def test_rolling(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 15, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 12, 15, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["/", ["rolling", "sum", 2,
                   ["metric", [str(self.metric.id), "mean"],
                    [str(metric2.id), "mean"]]], 2],
            granularities=[numpy.timedelta64(5, 'm')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 5, 0),
                          numpy.timedelta64(5, 'm'), 55.5),
                         (datetime64(2014, 1, 1, 12, 10, 0),
                          numpy.timedelta64(5, 'm'), 23),
                         (datetime64(2014, 1, 1, 12, 15, 0),
                          numpy.timedelta64(5, 'm'), 24)]
            },
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 5, 0),
                          numpy.timedelta64(5, 'm'), 5.5),
                         (datetime64(2014, 1, 1, 12, 10, 0),
                          numpy.timedelta64(5, 'm'), 3),
                         (datetime64(2014, 1, 1, 12, 15, 0),
                          numpy.timedelta64(5, 'm'), 4)]
            }
        }, values)

    def test_binary_operator_with_two_references(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["*", ["metric", str(self.metric.id), "mean"],
                  ["metric", str(metric2.id), "mean"]],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 621),
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 84),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 16),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 176),
        ], values)

    def test_binary_operator_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["*", ["metric", str(self.metric.id), "mean"], 2],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), 138),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 84),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 8),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 88)]
        }}, values)

    def test_ternary_operator_clip_min_max_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["clip", ["metric", str(self.metric.id), "mean"], 5, 60],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), 60),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 42),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 5),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 44)]
        }}, values)

    def test_ternary_operator_clip_min_ts_on_left(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["clip", ["metric", str(self.metric.id), "mean"], 50],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), 69),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 50),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 50),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 50)]
        }}, values)

    def test_binary_operator_ts_on_right(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.trigger_processing()

        values = processor.get_measures(
            self.storage, [processor.MetricReference(self.metric, "mean")],
            ["*", 2, ["metric", str(self.metric.id), "mean"]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({str(self.metric.id): {
            "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                      numpy.timedelta64(1, 'h'), 138),
                     (datetime64(2014, 1, 1, 13, 0, 0),
                      numpy.timedelta64(1, 'h'), 84),
                     (datetime64(2014, 1, 1, 14, 0, 0),
                      numpy.timedelta64(1, 'h'), 8),
                     (datetime64(2014, 1, 1, 15, 0, 0),
                      numpy.timedelta64(1, 'h'), 88)]
        }}, values)

    def test_mix(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            [
                "rolling",
                "sum",
                2,
                ["*", ["metric", str(self.metric.id), "mean"],
                      ["metric", str(metric2.id), "mean"]],
            ],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]

        self.assertEqual([
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 705),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 100),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 192),
        ], values)

    def test_bool(self):
        metric2, __ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), 9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), 2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), 4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            [
                "gt",
                [
                    "/",
                    [
                        "*",
                        ["*", ["metric", str(self.metric.id), "mean"],
                              ["metric", str(metric2.id), "mean"]],
                        100,
                    ],
                    1000
                ],
                10
            ],
            granularities=[numpy.timedelta64(1, 'h')])["aggregated"]
        self.assertEqual([
            (datetime64(2014, 1, 1, 12, 0, 0),
             numpy.timedelta64(1, 'h'), 1),
            (datetime64(2014, 1, 1, 13, 0, 0),
             numpy.timedelta64(1, 'h'), 0),
            (datetime64(2014, 1, 1, 14, 0, 0),
             numpy.timedelta64(1, 'h'), 0),
            (datetime64(2014, 1, 1, 15, 0, 0),
             numpy.timedelta64(1, 'h'), 1),
        ], values)

    def test_unary_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), -69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), -4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), -9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), -2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), -4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            ["abs", ["metric", [str(self.metric.id), "mean"],
                     [str(metric2.id), "mean"]]],
            granularities=[numpy.timedelta64(1, 'h')])

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                          numpy.timedelta64(1, 'h'), 69),
                         (datetime64(2014, 1, 1, 13, 0, 0),
                          numpy.timedelta64(1, 'h'), 42),
                         (datetime64(2014, 1, 1, 14, 0, 0),
                          numpy.timedelta64(1, 'h'), 4),
                         (datetime64(2014, 1, 1, 15, 0, 0),
                          numpy.timedelta64(1, 'h'), 44)]},
            str(metric2.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                          numpy.timedelta64(1, 'h'), 9),
                         (datetime64(2014, 1, 1, 13, 0, 0),
                          numpy.timedelta64(1, 'h'), 2),
                         (datetime64(2014, 1, 1, 14, 0, 0),
                          numpy.timedelta64(1, 'h'), 4),
                         (datetime64(2014, 1, 1, 15, 0, 0),
                          numpy.timedelta64(1, 'h'), 4),
                         ]}
        }, values)

    def test_max_operator(self):
        metric2, _ = self._create_metric()
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), -69),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 31), 42),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), -4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 45), 44),
        ])
        self.incoming.add_measures(metric2.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 5), -9),
            incoming.Measure(datetime64(2014, 1, 1, 13, 1, 41), -2),
            incoming.Measure(datetime64(2014, 1, 1, 14, 2, 31), 4),
            incoming.Measure(datetime64(2014, 1, 1, 15, 3, 10), -4),
        ])
        self.trigger_processing([self.metric, metric2])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean"),
             processor.MetricReference(metric2, "mean")],
            [
                "max",
                ["metric", str(self.metric.id), "mean"],
                [
                    "*",
                    ["metric", str(metric2.id), "mean"],
                    2,
                ]
            ],
            granularities=[numpy.timedelta64(1, 'h')]
        )["aggregated"]

        self.assertEqual(
            [
                (datetime64(2014, 1, 1, 12, 0, 0),
                 numpy.timedelta64(1, 'h'), -18),
                (datetime64(2014, 1, 1, 13, 0, 0),
                 numpy.timedelta64(1, 'h'), 42),
                (datetime64(2014, 1, 1, 14, 0, 0),
                 numpy.timedelta64(1, 'h'), 8),
                (datetime64(2014, 1, 1, 15, 0, 0),
                 numpy.timedelta64(1, 'h'), 44)
            ], values)

    def test_rateofchangesec_operator(self):
        self.incoming.add_measures(self.metric.id, [
            incoming.Measure(datetime64(2014, 1, 1, 12, 0, 1), 0),
            incoming.Measure(datetime64(2014, 1, 1, 12, 5, 10), 450),
            incoming.Measure(datetime64(2014, 1, 1, 12, 10, 5), 900),
            incoming.Measure(datetime64(2014, 1, 1, 12, 15, 30), 1350),
        ])
        self.trigger_processing([self.metric])

        values = processor.get_measures(
            self.storage,
            [processor.MetricReference(self.metric, "mean")],
            ["rateofchangesec", ["metric", str(self.metric.id), "mean"]],
            granularities=[numpy.timedelta64(5, 'm')],
        )

        self.assertEqual({
            str(self.metric.id): {
                "mean": [(datetime64(2014, 1, 1, 12, 0, 0),
                          numpy.timedelta64(5, 'm'), eq_nan),
                         (datetime64(2014, 1, 1, 12, 5, 0),
                          numpy.timedelta64(5, 'm'), 1.5),
                         (datetime64(2014, 1, 1, 12, 10, 0),
                          numpy.timedelta64(5, 'm'), 1.5),
                         (datetime64(2014, 1, 1, 12, 15, 0),
                          numpy.timedelta64(5, 'm'), 1.5),
                         ]}
        }, values)
