Merge pull request #12 from zangrcar/main

Refactoring

Author: jmrozanec

core/model.py

@@ -29,10 +29,11 @@ def with_preprocessing(self, strategy):
 class TimeseriesPreprocessing:
     """Processes timeseries."""
     def __init__(self):
-        pass
+        self.ts = None
 
     def process(self, ts):
-        pass
+        self.ts = ts
+        return TimeseriesView([ts])
 
 
 class TimeseriesPreprocessingSegmentation(TimeseriesPreprocessing):
@@ -48,6 +49,7 @@ class TimeseriesPreprocessingSegmentation(TimeseriesPreprocessing):
     def __init__(self, segment_start, segment_end):
         self.seg_st = segment_start
         self.seg_end = segment_end
+        self.ts = None
 
     def process(self, ts):
         self.ts = ts[self.seg_st:self.seg_end]
@@ -90,7 +92,7 @@ def __init__(self):
         self.segments = None
         self.strategy = []
 
-    def add_strategy(self, strat):
+    def add(self, strat):
         self.strategy.append(strat)
         return self
 
@@ -238,7 +240,7 @@ def _hash(self):
         str_to_hash = str(self.graph.nodes()) + str(self.graph.edges())
         return hashlib.md5(str_to_hash.encode()).hexdigest()
 
-    def combine_identical_nodes_slid_win(self):
+    def combine_identical_subgraphs(self):
         """Combines nodes that have same value of attribute self.attribute if graph is classical graph and
         nodes that are identical graphs if graph is created using sliding window mechanism."""
         self.orig_graph = self.graph.copy()
@@ -254,11 +256,11 @@ def combine_identical_nodes_slid_win(self):
                         continue
 
                     if(set(list(node_1.edges)) == set(list(node_2.edges))):
-                        self.graph = self._combine_nodes_win(self.graph, node_1, node_2, self.attribute)
+                        self.graph = self._combine_subgraphs(self.graph, node_1, node_2, self.attribute)
 
         return self
 
-    def _combine_nodes_win(self, graph, node_1, node_2, att):
+    def _combine_subgraphs(self, graph, node_1, node_2, att):
         """Combines nodes node_1 and node_2, that are graphs."""
         for i in range(len(list(node_1.nodes(data=True)))):
             for j in range(len(list(node_2.nodes(data=True))[i][1][att])):
@@ -446,7 +448,7 @@ def to_sequence(self, graph):
         self.node_strategy.set_arguments(self.graph, self.nodes, dictionaries, self.att)
 
 
-        i = 0
+        ts_len = 0
         while len(self.sequences[0]) < self.timeseries_len:
             for j in range(len(self.sequences)):
 
@@ -463,12 +465,12 @@ def to_sequence(self, graph):
             for j in range(self.skip_values + 1):
                 for k in range(len(current_nodes)):
 
-                    current_nodes[k] = self.node_strategy.next_node(i, k, current_nodes, self.switch_graphs, current_nodes[0])
+                    current_nodes[k] = self.node_strategy.next_node(ts_len, k, current_nodes, self.switch_graphs, current_nodes[0])
 
                     if(current_nodes[k] == None):
                         return self
 
-            i += 1
+            ts_len += 1
         return self.sequences
 
 class ToSequenceVisitor(ToSequenceVisitorMaster):

docs/documentation/input_output.md docs/documentation/tsg_io.mddocs/documentation/input_output.md renamed to docs/documentation/tsg_io.md

@@ -1,3 +1,3 @@
 ## **Csv and xml files**:
 
-::: input_output.input
+::: tsg_io.input

docs/tutorial.ipynb

@@ -1,4 +1,5 @@
 import random
+import numpy as np
 
 class StrategyNextValueInNode:
     """
@@ -103,6 +104,9 @@ def get_name(self):
         return "round robin"
 
 
+
+
+
 class StrategySelectNextNode:
     """
     Stores strategy to chose next node from the neighbors of the previous node.
@@ -141,7 +145,7 @@ def get_name(self):
         pass
 
 
-class StrategySelectNextNodeRandomlyAcrossGraphs(StrategySelectNextNode):
+class StrategySelectNextNodeRandomlyFromNeighboursAcrossGraphs(StrategySelectNextNode):
     """Walks through all graphs in a multivariate graph and chooses next node randomly."""
     def __init__(self):
         super().__init__()
@@ -155,10 +159,10 @@ def next_node(self, i, graph_index, nodes, switch, node):
         return random.choice(neighbors)
 
     def get_name(self):
-        return "walkthrough all graphs randomly"
+        return "walkthrough all graphs randomly from neighbours"
 
 
-class StrategySelectNextNodeRandomlyFromFirstGraph(StrategySelectNextNode):
+class StrategySelectNextNodeRandomlyFromNeighboursFromFirstGraph(StrategySelectNextNode):
     """Walks through first graph and chooses next node randomly."""
     def __init__(self):
         super().__init__()
@@ -171,4 +175,64 @@ def next_node(self, i, graph_index, nodes, switch, node):
         return random.choice(neighbors)
 
     def get_name(self):
-        return "walkthrough one graph randomly"
+        return "walkthrough one graph randomly from neighbours"
+
+#TODO: fix this
+class StrategySelectNextNodeRandomly(StrategySelectNextNode):
+    
+    def __init__(self):
+        super().__init__()
+    
+    def next_node(self, i, graph_index, nodes, switch, node):
+        return random.choice(self.nodes[graph_index])
+    
+    def get_name(self):
+        return "Random walkthrough the nodes"
+    
+
+#TODO: fix this
+class StrategySelectNextNodeRandomDegree(StrategySelectNextNode):
+
+    def __init__(self):
+        super().__init__()
+
+    def next_node(self, i, graph_index, nodes, switch, node):
+        nodes_weighted_tuples = [(n, float(len([x for x in list(set(self.nodes[graph_index]) & set(self.graph.neighbors(n)))]))/float(len(nodes[graph_index]))) for n in nodes[graph_index]]
+        nodes = [n[0] for n in nodes_weighted_tuples]
+        node_weights = [n[1] for n in nodes_weighted_tuples]
+        if np.min(node_weights)>0:
+            node_weights = np.round(np.divide(node_weights, np.min(node_weights)), decimals=4)
+        node_weights = np.divide(node_weights, np.sum(node_weights))
+
+        return np.random.choice(nodes, p=node_weights)
+    
+
+    def get_name(self):
+        return "Random degree walkthrough the nodes"
+    
+#TODO: fix this
+class StrategySelectNextNodeRandomWithRestart(StrategySelectNextNode):
+    
+    def __init__(self):
+        super().__init__()
+        self.first_node = None
+    
+    def next_node(self, i, graph_index, nodes, switch, node):
+        
+        if self.first_node == None:
+            self.first_node = []
+            for i in range(len(nodes)):
+                self.first_node.append(np.random.choice(nodes[i]))
+
+        if np.random.random() <0.15:
+            return self.first_node[graph_index]
+        
+        if len(nodes) == 0:
+            node = self.first_node[graph_index]
+        else:
+            node = np.random.choice(nodes[graph_index])
+
+        return node
+    
+    def get_name(self):
+        return "Random walk with restart"

from_graph/strategy_to_time_sequence.py

@@ -7,16 +7,6 @@ class GraphToTimeseriesStrategy:
     def to_sequence(self, graph, sequence_length):
         return None
 
-class RandomNodeSequenceGenerationStrategy(GraphToTimeseriesStrategy):
-    def to_sequence(self, graph, sequence_length):
-        sequence = []
-        nodes = [n for n in graph.nodes()]
-
-        while len(sequence) < sequence_length:
-            node = np.random.choice(nodes)
-            sequence = sequence + [graph.nodes[node]['value']]
-
-        return sequence
 
 class RandomNodeNeighbourSequenceGenerationStrategy(GraphToTimeseriesStrategy):
     def to_sequence(self, graph, sequence_length):

generation/strategies.py

@@ -81,7 +81,7 @@ nav:
   - Home: index.md
 
   - Documentation:
-    - Input/output: documentation/input_output.md
+    - Input/output: documentation/tsg_io.md
     - Model: documentation/model.md
     - To Time sequence strategy: documentation/strategy_to_time_sequence.md
     - Graph linking strategy: documentation/strategy_linking_graph.md

mkdocs.yml

@@ -21,8 +21,6 @@ def __init__(self, graph, strategy_precedence):
     def set_graph(self, graph):
         self.graph = graph
 
-    #TODO: implement/make it comparable :)
-    #TODO: rename :)
     def get_strategy_precedence(self):
         return self.strategy_precedence
 

to_graph/strategy_linking_graph.py

@@ -10,7 +10,7 @@ def __init__(self):
     def from_csv(self):
         pass
 
-class CsvStock(CsvRead):
+class CsvFile(CsvRead):
     """
     Returns proccessed data from csv file sorted by "Date".
     
@@ -41,10 +41,11 @@ def __init__(self):
     def from_xml(self):
         pass
 
-#TODO: rename
-class XmlSomething(XmlRead):
+class FundamentalsReportFinancialStatements(XmlRead):
     """
-    One of the ways of extracting the data from xml file.
+    Extracting data from an xml file.
+    Further explanation on file format can be found on:
+    https://docs-2-0--quantrocket.netlify.app/data/reuters/
     
     **Attributes:**
 

input_output/input.py tsg_io/input.pyinput_output/input.py renamed to tsg_io/input.py

@@ -4,90 +4,90 @@
 if nb_dir not in sys.path:
     sys.path.append(nb_dir)
 
-from core import model
+from core.model import Timeseries, TimeseriesPreprocessing, TimeseriesPreprocessingSegmentation, TimeseriesPreprocessingSlidingWindow, TimeseriesPreprocessingComposite, TimeseriesView, TimeGraph, ToSequenceVisitorSlidingWindow, ToSequenceVisitor
 
-from input_output import input as inp
-import from_graph.strategy_to_time_sequence as tts
-import to_graph.strategy_linking_graph as gls
-import to_graph.strategy_linking_multi_graphs as mgl
-import to_graph.strategy_to_graph as tgs
+from tsg_io.input import CsvFile
+from from_graph.strategy_to_time_sequence import StrategyNextValueInNodeRandom, StrategyNextValueInNodeRandomForSlidingWindow, StrategyNextValueInNodeRoundRobin, StrategyNextValueInNodeRoundRobinForSlidingWindow, StrategySelectNextNodeRandomlyFromNeighboursAcrossGraphs, StrategySelectNextNodeRandomlyFromNeighboursFromFirstGraph, StrategySelectNextNodeRandomly, StrategySelectNextNodeRandomDegree, StrategySelectNextNodeRandomWithRestart
+from to_graph.strategy_linking_graph import StrategyLinkingGraphByValueWithinRange, LinkNodesWithinGraph
+from to_graph.strategy_linking_multi_graphs import LinkGraphs
+from to_graph.strategy_to_graph import BuildTimeseriesToGraphNaturalVisibilityStrategy, BuildTimeseriesToGraphHorizontalVisibilityStrategy
 
 amazon_path = os.path.join(os.getcwd(), "amazon", "AMZN.csv")
 apple_path = os.path.join(os.getcwd(), "apple", "APPLE.csv")
 
 
-test = model.Timeseries(inp.CsvStock(amazon_path, "Close").from_csv())\
-    .with_preprocessing(model.TimeseriesPreprocessingSegmentation(60, 90))\
-    .to_graph(tgs.BuildTimeseriesToGraphNaturalVisibilityStrategy().with_limit(1).get_strategy())\
+
+timegraph_1 = Timeseries(CsvFile(amazon_path, "Close").from_csv())\
+    .with_preprocessing(TimeseriesPreprocessingSegmentation(60, 90))\
+    .to_graph(BuildTimeseriesToGraphNaturalVisibilityStrategy().with_limit(1).get_strategy())\
     .add_edge(0,2)\
     .add_edge(13, 21, weight = 17)\
-    .link(gls.LinkNodesWithinGraph().by_value(gls.StrategyLinkingGraphByValueWithinRange(2)).seasonalities(15))\
+    .link(LinkNodesWithinGraph().by_value(StrategyLinkingGraphByValueWithinRange(2)).seasonalities(15))\
     .draw("blue")
 
-
-x = model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-    .with_preprocessing(model.TimeseriesPreprocessingComposite()\
-        .add_strategy(model.TimeseriesPreprocessingSegmentation(60, 120))\
-        .add_strategy(model.TimeseriesPreprocessingSlidingWindow(5)))\
-    .to_graph(tgs.BuildTimeseriesToGraphNaturalVisibilityStrategy().get_strategy())\
-    .link(mgl.LinkGraphs().sliding_window())\
-    .combine_identical_nodes_slid_win()\
+timegraph_2 = Timeseries(CsvFile(apple_path, "Close").from_csv())\
+    .with_preprocessing(TimeseriesPreprocessingComposite()\
+        .add(TimeseriesPreprocessingSegmentation(60, 120))\
+        .add(TimeseriesPreprocessingSlidingWindow(5)))\
+    .to_graph(BuildTimeseriesToGraphNaturalVisibilityStrategy().get_strategy())\
+    .link(LinkGraphs().sliding_window())\
+    .combine_identical_subgraphs()\
     .draw("red")
 
-i = model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-    .with_preprocessing(model.TimeseriesPreprocessingSegmentation(60, 90))\
-    .add(model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-        .with_preprocessing(model.TimeseriesPreprocessingSegmentation(90, 120)))\
-    .add(model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-        .with_preprocessing(model.TimeseriesPreprocessingSegmentation(150, 180)))\
-    .to_graph(tgs.BuildTimeseriesToGraphNaturalVisibilityStrategy().with_limit(1).get_strategy())\
-    .link(mgl.LinkGraphs().time_cooccurrence())\
-    .link(gls.LinkNodesWithinGraph().by_value(gls.StrategyLinkingGraphByValueWithinRange(0.5)))\
+timegraph_3 = Timeseries(CsvFile(apple_path, "Close").from_csv())\
+    .with_preprocessing(TimeseriesPreprocessingSegmentation(60, 90))\
+    .add(Timeseries(CsvFile(apple_path, "Close").from_csv())\
+        .with_preprocessing(TimeseriesPreprocessingSegmentation(90, 120)))\
+    .add(Timeseries(CsvFile(apple_path, "Close").from_csv())\
+        .with_preprocessing(TimeseriesPreprocessingSegmentation(150, 180)))\
+    .to_graph(BuildTimeseriesToGraphNaturalVisibilityStrategy().with_limit(1).get_strategy())\
+    .link(LinkGraphs().time_cooccurrence())\
+    .link(LinkNodesWithinGraph().by_value(StrategyLinkingGraphByValueWithinRange(0.5)))\
     .combine_identical_nodes()\
     .draw("brown")
 
-j = model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-    .with_preprocessing(model.TimeseriesPreprocessingComposite()\
-        .add_strategy(model.TimeseriesPreprocessingSegmentation(60, 110))\
-        .add_strategy(model.TimeseriesPreprocessingSlidingWindow(5)))\
-    .add(model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-        .with_preprocessing(model.TimeseriesPreprocessingComposite()\
-            .add_strategy(model.TimeseriesPreprocessingSegmentation(120, 170))\
-            .add_strategy(model.TimeseriesPreprocessingSlidingWindow(5)))\
-        .add(model.Timeseries(inp.CsvStock(apple_path, "Close").from_csv())\
-            .with_preprocessing(model.TimeseriesPreprocessingComposite()\
-                    .add_strategy(model.TimeseriesPreprocessingSegmentation(190, 240))\
-                    .add_strategy(model.TimeseriesPreprocessingSlidingWindow(5)))))\
-    .to_graph(tgs.BuildTimeseriesToGraphNaturalVisibilityStrategy().get_strategy())\
-    .link(mgl.LinkGraphs().sliding_window().time_cooccurrence())\
-    .combine_identical_nodes_slid_win()\
-    .link(gls.LinkNodesWithinGraph().seasonalities(15))\
+timegraph_4 = Timeseries(CsvFile(apple_path, "Close").from_csv())\
+    .with_preprocessing(TimeseriesPreprocessingComposite()\
+        .add(TimeseriesPreprocessingSegmentation(60, 110))\
+        .add(TimeseriesPreprocessingSlidingWindow(5)))\
+    .add(Timeseries(CsvFile(apple_path, "Close").from_csv())\
+        .with_preprocessing(TimeseriesPreprocessingComposite()\
+            .add(TimeseriesPreprocessingSegmentation(120, 170))\
+            .add(TimeseriesPreprocessingSlidingWindow(5)))\
+        .add(Timeseries(CsvFile(apple_path, "Close").from_csv())\
+            .with_preprocessing(TimeseriesPreprocessingComposite()\
+                    .add(TimeseriesPreprocessingSegmentation(190, 240))\
+                    .add(TimeseriesPreprocessingSlidingWindow(5)))))\
+    .to_graph(BuildTimeseriesToGraphNaturalVisibilityStrategy().get_strategy())\
+    .link(LinkGraphs().sliding_window().time_cooccurrence())\
+    .combine_identical_subgraphs()\
+    .link(LinkNodesWithinGraph().seasonalities(15))\
     .draw("green")
 
 
-test.to_sequence(model.ToSequenceVisitor()\
-        .next_node_strategy(tts.StrategySelectNextNodeRandomlyAcrossGraphs())\
-        .next_value_strategy(tts.StrategyNextValueInNodeRoundRobin().skip_every_x_steps(1))\
+timegraph_1.to_sequence(ToSequenceVisitor()\
+        .next_node_strategy(StrategySelectNextNodeRandomWithRestart())\
+        .next_value_strategy(StrategyNextValueInNodeRoundRobin().skip_every_x_steps(1))\
         .ts_length(100))\
     .draw_sequence()
 
 
-x.to_sequence(model.ToSequenceVisitorSlidingWindow()\
-    .next_node_strategy(tts.StrategySelectNextNodeRandomlyFromFirstGraph())\
-    .next_value_strategy(tts.StrategyNextValueInNodeRandomForSlidingWindow().skip_every_x_steps(1))\
+timegraph_2.to_sequence(ToSequenceVisitorSlidingWindow()\
+    .next_node_strategy(StrategySelectNextNodeRandomly())\
+    .next_value_strategy(StrategyNextValueInNodeRandomForSlidingWindow().skip_every_x_steps(1))\
     .ts_length(50))\
     .draw_sequence()
 
 
-i.to_sequence(model.ToSequenceVisitor()\
-    .next_node_strategy(tts.StrategySelectNextNodeRandomlyAcrossGraphs().change_graphs_every_x_steps(2))\
-    .next_value_strategy(tts.StrategyNextValueInNodeRoundRobin().skip_every_x_steps(1))\
+timegraph_3.to_sequence(ToSequenceVisitor()\
+    .next_node_strategy(StrategySelectNextNodeRandomlyFromNeighboursAcrossGraphs().change_graphs_every_x_steps(2))\
+    .next_value_strategy(StrategyNextValueInNodeRoundRobin().skip_every_x_steps(1))\
     .ts_length(50))\
     .draw_sequence()
 
 
-j.to_sequence(model.ToSequenceVisitorSlidingWindow()\
-    .next_node_strategy(tts.StrategySelectNextNodeRandomlyFromFirstGraph())\
-    .next_value_strategy(tts.StrategyNextValueInNodeRoundRobinForSlidingWindow())\
+timegraph_4.to_sequence(ToSequenceVisitorSlidingWindow()\
+    .next_node_strategy(StrategySelectNextNodeRandomlyFromNeighboursAcrossGraphs())\
+    .next_value_strategy(StrategyNextValueInNodeRoundRobinForSlidingWindow())\
     .ts_length(100))\
     .draw_sequence()