cuda_autoencoder/prep_layer.py at master · c0stya/cuda_autoencoder · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
"""
Preprocess dataset for sent140 corpus.

"""
import numpy as np
import sys

from utils import alph, translate, ALPH_rev

def encode(data, alph):
    m, n = data.shape # batch_size x n_steps

    enc = np.zeros((m,n*alph), dtype=np.uint8)

    for i in xrange(m):
        for j in xrange(n):
            enc[i, j*alph + data[i,j]] = 1

    return enc

if __name__ == '__main__':

    if len(sys.argv) < 3:
        print "Usage: %s <input_corpus> <output>" % (sys.argv[0])
        sys.exit(1)

    inp = sys.argv[1]
    out = sys.argv[2]

    with open(inp) as h:
        alph = np.load(h)[0]
        X = np.load(h)
        Y = np.load(h)

    # this could be memory-critical part
    enc_X = encode(X, alph)

    with open(out, 'wb') as h:
        np.save(h, enc_X) # indicates the layer