Skip to content
Snippets Groups Projects
Select Git revision
  • main default protected
  • decision-tree-demo
2 results
Blame Permalink
data.py 2.75 KiB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99 






#!/usr/bin/env python
#-*- coding:utf-8 -*-
##
## data.py
##
##  Created on: Sep 20, 2017
##      Author: Alexey Ignatiev, Nina Narodytska
##      E-mail: aignatiev@ciencias.ulisboa.pt, narodytska@vmware.com
##

#
#==============================================================================
from __future__ import print_function
import collections
import itertools
import pickle
import six
import gzip
from six.moves import range
import numpy as np
import pandas as pd

#
#==============================================================================
class Data(object):
    """
        Class for representing data (transactions).
    """

    def __init__(self, data, separator=','):
        """
            Constructor and parser.
        """
        self.names = None
        self.nm2id = None
        self.feats = None
        self.targets = None
        self.samples = None

        self.parse(data, separator)
           
    def parse(self, data, separator):
        """
            Parse input file.
        """

        # reading data set from file
        lines = data.split('\n')

        # reading preamble
        self.names = [name.replace('"','').strip() for name in lines[0].strip().split(separator)]
        self.feats = [set([]) for n in self.names[:-1]]
        self.targets = set([])
        
        lines = lines[1:]

        # filling name to id mapping
        self.nm2id = {name: i for i, name in enumerate(self.names)}

        self.nonbin2bin = {}
        for name in self.nm2id:
            spl = name.rsplit(':',1)
            if (spl[0] not in self.nonbin2bin):
                self.nonbin2bin[spl[0]] = [name]
            else:
                self.nonbin2bin[spl[0]].append(name)

        # reading training samples
        self.samples =  []

        for line, w in six.iteritems(collections.Counter(lines)):
            inst = [v.strip() for v in line.strip().split(separator)]
            self.samples.append(inst)
            for i, v in enumerate(inst[:-1]):
                if v:
                    self.feats[i].add(str(v)) 
            assert(inst[-1])
            self.targets.add(str(inst[-1]))
        
        self.nof_feats = len(self.names[:-1])            
        
    def mapping_features(self):
        """
            feature-value mapping
        """
        fvmap = {}
        
        for i in range(self.nof_feats):
            fvmap[f'f{i}'] = dict() 
            for j, v in enumerate(sorted(self.feats[i])):
                fvmap[f'f{i}'][j] = (self.names[i], True, v)
            
            if len(self.feats[i]) > 2:
                m = len(self.feats[i])
                for j, v in enumerate(sorted(self.feats[i])):
                    fvmap[f'f{i}'][j+m] = (self.names[i], False, v)                                
            
        return fvmap