Browse code

run pre-commit run --all

Bas Nijholt authored on 11/12/2019 14:00:39
Showing 1 changed files
... ...
@@ -1,5 +1,3 @@
1
-# -*- coding: utf-8 -*-
2
-
3 1
 import abc
4 2
 import functools
5 3
 import gzip
Browse code

implement Joe's suggestion

Bas Nijholt authored on 17/10/2019 09:52:30
Showing 1 changed files
... ...
@@ -76,10 +76,12 @@ class _RequireAttrsABCMeta(abc.ABCMeta):
76 76
         for name, type_ in obj.__annotations__.items():
77 77
             try:
78 78
                 x = getattr(obj, name)
79
-                if not isinstance(x, type_):
80
-                    raise TypeError(
81
-                        f"The attribute '{name}' is of {type_} instead of {type(x)}."
82
-                    )
83 79
             except AttributeError:
84
-                raise AttributeError(f"Required attribute {name} not set in __init__.")
80
+                raise AttributeError(
81
+                    f"Required attribute {name} not set in __init__."
82
+                ) from None
83
+            else:
84
+                if not isinstance(x, type_):
85
+                    msg = f"The attribute '{name}' should be of type {type_}, not {type(x)}."
86
+                    raise TypeError(msg)
85 87
         return obj
Browse code

fix illegal syntax

Bas Nijholt authored on 17/10/2019 09:20:34
Showing 1 changed files
... ...
@@ -80,6 +80,6 @@ class _RequireAttrsABCMeta(abc.ABCMeta):
80 80
                     raise TypeError(
81 81
                         f"The attribute '{name}' is of {type_} instead of {type(x)}."
82 82
                     )
83
-            except AttributeError as e:
84
-                raise e(f"Required attribute {name} not set in __init__.")
83
+            except AttributeError:
84
+                raise AttributeError(f"Required attribute {name} not set in __init__.")
85 85
         return obj
Browse code

use a try except in _RequireAttrsABCMeta

Bas Nijholt authored on 16/10/2019 22:20:20
Showing 1 changed files
... ...
@@ -74,11 +74,12 @@ class _RequireAttrsABCMeta(abc.ABCMeta):
74 74
     def __call__(self, *args, **kwargs):
75 75
         obj = super().__call__(*args, **kwargs)
76 76
         for name, type_ in obj.__annotations__.items():
77
-            if not hasattr(obj, name):
78
-                raise ValueError(f"Required attribute {name} not set in __init__.")
79
-            elif not isinstance(getattr(obj, name), type_):
80
-                raise TypeError(
81
-                    f"The attribute '{name}' is of {type_} instead of {type(getattr(obj, name))}"
82
-                )
83
-
77
+            try:
78
+                x = getattr(obj, name)
79
+                if not isinstance(x, type_):
80
+                    raise TypeError(
81
+                        f"The attribute '{name}' is of {type_} instead of {type(x)}."
82
+                    )
83
+            except AttributeError as e:
84
+                raise e(f"Required attribute {name} not set in __init__.")
84 85
         return obj
Browse code

use type annotations

Bas Nijholt authored on 16/10/2019 20:41:24
Showing 1 changed files
... ...
@@ -71,11 +71,14 @@ def copy_docstring_from(other):
71 71
 
72 72
 
73 73
 class _RequireAttrsABCMeta(abc.ABCMeta):
74
-    required_attributes = []
75
-
76 74
     def __call__(self, *args, **kwargs):
77 75
         obj = super().__call__(*args, **kwargs)
78
-        for name in obj.required_attributes:
76
+        for name, type_ in obj.__annotations__.items():
79 77
             if not hasattr(obj, name):
80 78
                 raise ValueError(f"Required attribute {name} not set in __init__.")
79
+            elif not isinstance(getattr(obj, name), type_):
80
+                raise TypeError(
81
+                    f"The attribute '{name}' is of {type_} instead of {type(getattr(obj, name))}"
82
+                )
83
+
81 84
         return obj
Browse code

add utils._RequireAttrsABCMeta

Bas Nijholt authored on 14/10/2019 12:57:58
Showing 1 changed files
... ...
@@ -1,5 +1,6 @@
1 1
 # -*- coding: utf-8 -*-
2 2
 
3
+import abc
3 4
 import functools
4 5
 import gzip
5 6
 import os
... ...
@@ -67,3 +68,14 @@ def copy_docstring_from(other):
67 68
         return functools.wraps(other)(method)
68 69
 
69 70
     return decorator
71
+
72
+
73
+class _RequireAttrsABCMeta(abc.ABCMeta):
74
+    required_attributes = []
75
+
76
+    def __call__(self, *args, **kwargs):
77
+        obj = super().__call__(*args, **kwargs)
78
+        for name in obj.required_attributes:
79
+            if not hasattr(obj, name):
80
+                raise ValueError(f"Required attribute {name} not set in __init__.")
81
+        return obj
Browse code

ensure atomic writes when saving a file

Right now, if a program crashes in the middle of saving, you lose
all your data.

Right now, if a program crashes in the middle of saving, you lose
all your data. This ensures that the old file is first moved, then
the new file is saved, and only then the old file is removed.

Bas Nijholt authored on 21/06/2019 00:04:07
Showing 1 changed files
... ...
@@ -7,6 +7,8 @@ import pickle
7 7
 from contextlib import contextmanager
8 8
 from itertools import product
9 9
 
10
+from atomicwrites import AtomicWriter
11
+
10 12
 
11 13
 def named_product(**items):
12 14
     names = items.keys()
... ...
@@ -44,9 +46,13 @@ def save(fname, data, compress=True):
44 46
     dirname = os.path.dirname(fname)
45 47
     if dirname:
46 48
         os.makedirs(dirname, exist_ok=True)
47
-    _open = gzip.open if compress else open
48
-    with _open(fname, "wb") as f:
49
-        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
49
+
50
+    blob = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)
51
+    if compress:
52
+        blob = gzip.compress(blob)
53
+
54
+    with AtomicWriter(fname, "wb", overwrite=True).open() as f:
55
+        f.write(blob)
50 56
 
51 57
 
52 58
 def load(fname, compress=True):
Browse code

fix all flake8 issues and run pre-commit filters

Bas Nijholt authored on 08/05/2019 02:26:15
Showing 1 changed files
... ...
@@ -4,7 +4,6 @@ import functools
4 4
 import gzip
5 5
 import os
6 6
 import pickle
7
-import time
8 7
 from contextlib import contextmanager
9 8
 from itertools import product
10 9
 
... ...
@@ -28,13 +27,15 @@ def restore(*learners):
28 27
 def cache_latest(f):
29 28
     """Cache the latest return value of the function and add it
30 29
     as 'self._cache[f.__name__]'."""
30
+
31 31
     @functools.wraps(f)
32 32
     def wrapper(*args, **kwargs):
33 33
         self = args[0]
34
-        if not hasattr(self, '_cache'):
34
+        if not hasattr(self, "_cache"):
35 35
             self._cache = {}
36 36
         self._cache[f.__name__] = f(*args, **kwargs)
37 37
         return self._cache[f.__name__]
38
+
38 39
     return wrapper
39 40
 
40 41
 
... ...
@@ -44,18 +45,19 @@ def save(fname, data, compress=True):
44 45
     if dirname:
45 46
         os.makedirs(dirname, exist_ok=True)
46 47
     _open = gzip.open if compress else open
47
-    with _open(fname, 'wb') as f:
48
+    with _open(fname, "wb") as f:
48 49
         pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
49 50
 
50 51
 
51 52
 def load(fname, compress=True):
52 53
     fname = os.path.expanduser(fname)
53 54
     _open = gzip.open if compress else open
54
-    with _open(fname, 'rb') as f:
55
+    with _open(fname, "rb") as f:
55 56
         return pickle.load(f)
56 57
 
57 58
 
58 59
 def copy_docstring_from(other):
59 60
     def decorator(method):
60 61
         return functools.wraps(other)(method)
62
+
61 63
     return decorator
Browse code

fix import order and style

Bas Nijholt authored on 07/05/2019 23:42:40
Showing 1 changed files
... ...
@@ -1,12 +1,12 @@
1 1
 # -*- coding: utf-8 -*-
2 2
 
3
-from contextlib import contextmanager
4 3
 import functools
5 4
 import gzip
6
-from itertools import product
7 5
 import os
8 6
 import pickle
9 7
 import time
8
+from contextlib import contextmanager
9
+from itertools import product
10 10
 
11 11
 
12 12
 def named_product(**items):
Browse code

adhere to PEP008 by using absolute imports

See https://www.python.org/dev/peps/pep-0008/#imports

Bas Nijholt authored on 26/11/2018 13:39:12
Showing 1 changed files
... ...
@@ -1,4 +1,5 @@
1 1
 # -*- coding: utf-8 -*-
2
+
2 3
 from contextlib import contextmanager
3 4
 import functools
4 5
 import gzip
Browse code

save execution time directly on the futures inside the runners

Previously we were wrapping the learner function and doing the
timing there. This makes things more complicated in the case that
we are learning an 'async def' function.

Joseph Weston authored on 25/10/2018 15:45:25
Showing 1 changed files
... ...
@@ -8,12 +8,6 @@ import pickle
8 8
 import time
9 9
 
10 10
 
11
-def timed(f, *args, **kwargs):
12
-    t_start = time.time()
13
-    result = f(*args, **kwargs)
14
-    return result, time.time() - t_start
15
-
16
-
17 11
 def named_product(**items):
18 12
     names = items.keys()
19 13
     vals = items.values()
Browse code

add 'save' and 'load' to the learners and periodic saving to the Runner

Bas Nijholt authored on 08/10/2018 17:08:34
Showing 1 changed files
... ...
@@ -1,7 +1,10 @@
1 1
 # -*- coding: utf-8 -*-
2 2
 from contextlib import contextmanager
3
-from functools import wraps
3
+import functools
4
+import gzip
4 5
 from itertools import product
6
+import os
7
+import pickle
5 8
 import time
6 9
 
7 10
 
... ...
@@ -30,7 +33,7 @@ def restore(*learners):
30 33
 def cache_latest(f):
31 34
     """Cache the latest return value of the function and add it
32 35
     as 'self._cache[f.__name__]'."""
33
-    @wraps(f)
36
+    @functools.wraps(f)
34 37
     def wrapper(*args, **kwargs):
35 38
         self = args[0]
36 39
         if not hasattr(self, '_cache'):
... ...
@@ -38,3 +41,26 @@ def cache_latest(f):
38 41
         self._cache[f.__name__] = f(*args, **kwargs)
39 42
         return self._cache[f.__name__]
40 43
     return wrapper
44
+
45
+
46
+def save(fname, data, compress=True):
47
+    fname = os.path.expanduser(fname)
48
+    dirname = os.path.dirname(fname)
49
+    if dirname:
50
+        os.makedirs(dirname, exist_ok=True)
51
+    _open = gzip.open if compress else open
52
+    with _open(fname, 'wb') as f:
53
+        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
54
+
55
+
56
+def load(fname, compress=True):
57
+    fname = os.path.expanduser(fname)
58
+    _open = gzip.open if compress else open
59
+    with _open(fname, 'rb') as f:
60
+        return pickle.load(f)
61
+
62
+
63
+def copy_docstring_from(other):
64
+    def decorator(method):
65
+        return functools.wraps(other)(method)
66
+    return decorator
Browse code

add 'cache_loss' decorator

Bas Nijholt authored on 08/10/2018 13:13:27
Showing 1 changed files
... ...
@@ -1,5 +1,6 @@
1 1
 # -*- coding: utf-8 -*-
2 2
 from contextlib import contextmanager
3
+from functools import wraps
3 4
 from itertools import product
4 5
 import time
5 6
 
... ...
@@ -24,3 +25,16 @@ def restore(*learners):
24 25
     finally:
25 26
         for state, learner in zip(states, learners):
26 27
             learner.__setstate__(state)
28
+
29
+
30
+def cache_latest(f):
31
+    """Cache the latest return value of the function and add it
32
+    as 'self._cache[f.__name__]'."""
33
+    @wraps(f)
34
+    def wrapper(*args, **kwargs):
35
+        self = args[0]
36
+        if not hasattr(self, '_cache'):
37
+            self._cache = {}
38
+        self._cache[f.__name__] = f(*args, **kwargs)
39
+        return self._cache[f.__name__]
40
+    return wrapper
Browse code

time 'learner.function' with a function instead of a class

Bas Nijholt authored on 20/06/2018 20:09:22
Showing 1 changed files
... ...
@@ -4,14 +4,10 @@ from itertools import product
4 4
 import time
5 5
 
6 6
 
7
-class TimeReturn:
8
-    def __init__(self, function):
9
-        self.function = function
10
-
11
-    def __call__(self, *args, **kwargs):
12
-        t_start = time.time()
13
-        result = self.function(*args, **kwargs)
14
-        return result, time.time() - t_start
7
+def timed(f, *args, **kwargs):
8
+    t_start = time.time()
9
+    result = f(*args, **kwargs)
10
+    return result, time.time() - t_start
15 11
 
16 12
 
17 13
 def named_product(**items):
Browse code

remove unneeded getstate and setstate methods

Bas Nijholt authored on 16/06/2018 23:17:59
Showing 1 changed files
... ...
@@ -5,21 +5,13 @@ import time
5 5
 
6 6
 
7 7
 class TimeReturn:
8
-    def __init__(self, function, total_time=0):
8
+    def __init__(self, function):
9 9
         self.function = function
10
-        self.total_time = total_time
11 10
 
12 11
     def __call__(self, *args, **kwargs):
13 12
         t_start = time.time()
14 13
         result = self.function(*args, **kwargs)
15
-        self.total_time += time.time() - t_start
16
-        return result, self.total_time
17
-
18
-    def __getstate__(self):
19
-        return (self.function, self.total_time)
20
-
21
-    def __setstate__(self, state):
22
-        self.__init__(*state)
14
+        return result, time.time() - t_start
23 15
 
24 16
 
25 17
 def named_product(**items):
Browse code

remove unused imports/classes

Bas Nijholt authored on 16/06/2018 23:14:17
Showing 1 changed files
... ...
@@ -4,24 +4,6 @@ from itertools import product
4 4
 import time
5 5
 
6 6
 
7
-class WithTime:
8
-    def __init__(self, function, time=0):
9
-        self.function = function
10
-        self.time = time
11
-
12
-    def __call__(self, *args, **kwargs):
13
-        t_start = time.time()
14
-        result = self.function(*args, **kwargs)
15
-        self.time += time.time() - t_start
16
-        return result
17
-
18
-    def __getstate__(self):
19
-        return (self.function, self.time)
20
-
21
-    def __setstate__(self, state):
22
-        self.__init__(*state)
23
-
24
-
25 7
 class TimeReturn:
26 8
     def __init__(self, function, total_time=0):
27 9
         self.function = function
Browse code

remove 'performance' and define 'overhead' (1 - efficiecy)

Bas Nijholt authored on 16/06/2018 23:02:26
Showing 1 changed files
... ...
@@ -22,26 +22,6 @@ class WithTime:
22 22
         self.__init__(*state)
23 23
 
24 24
 
25
-class AverageTimeReturn:
26
-    def __init__(self, function, total_time=0, n=0):
27
-        self.function = function
28
-        self.total_time = total_time
29
-        self.n = n
30
-
31
-    def __call__(self, *args, **kwargs):
32
-        t_start = time.time()
33
-        result = self.function(*args, **kwargs)
34
-        self.total_time += time.time() - t_start
35
-        self.n += 1
36
-        return result, self.total_time / self.n
37
-
38
-    def __getstate__(self):
39
-        return (self.function, self.total_time, self.n)
40
-
41
-    def __setstate__(self, state):
42
-        self.__init__(*state)
43
-
44
-
45 25
 class TimeReturn:
46 26
     def __init__(self, function, total_time=0):
47 27
         self.function = function
Browse code

calculate the actual efficiency of the resources

Bas Nijholt authored on 16/06/2018 22:32:07
Showing 1 changed files
... ...
@@ -42,6 +42,24 @@ class AverageTimeReturn:
42 42
         self.__init__(*state)
43 43
 
44 44
 
45
+class TimeReturn:
46
+    def __init__(self, function, total_time=0):
47
+        self.function = function
48
+        self.total_time = total_time
49
+
50
+    def __call__(self, *args, **kwargs):
51
+        t_start = time.time()
52
+        result = self.function(*args, **kwargs)
53
+        self.total_time += time.time() - t_start
54
+        return result, self.total_time
55
+
56
+    def __getstate__(self):
57
+        return (self.function, self.total_time)
58
+
59
+    def __setstate__(self, state):
60
+        self.__init__(*state)
61
+
62
+
45 63
 def named_product(**items):
46 64
     names = items.keys()
47 65
     vals = items.values()
Browse code

correctly keep track of the average time

Bas Nijholt authored on 16/06/2018 09:07:44
Showing 1 changed files
... ...
@@ -1,6 +1,45 @@
1 1
 # -*- coding: utf-8 -*-
2 2
 from contextlib import contextmanager
3 3
 from itertools import product
4
+import time
5
+
6
+
7
+class WithTime:
8
+    def __init__(self, function, time=0):
9
+        self.function = function
10
+        self.time = time
11
+
12
+    def __call__(self, *args, **kwargs):
13
+        t_start = time.time()
14
+        result = self.function(*args, **kwargs)
15
+        self.time += time.time() - t_start
16
+        return result
17
+
18
+    def __getstate__(self):
19
+        return (self.function, self.time)
20
+
21
+    def __setstate__(self, state):
22
+        self.__init__(*state)
23
+
24
+
25
+class AverageTimeReturn:
26
+    def __init__(self, function, total_time=0, n=0):
27
+        self.function = function
28
+        self.total_time = total_time
29
+        self.n = n
30
+
31
+    def __call__(self, *args, **kwargs):
32
+        t_start = time.time()
33
+        result = self.function(*args, **kwargs)
34
+        self.total_time += time.time() - t_start
35
+        self.n += 1
36
+        return result, self.total_time / self.n
37
+
38
+    def __getstate__(self):
39
+        return (self.function, self.total_time, self.n)
40
+
41
+    def __setstate__(self, state):
42
+        self.__init__(*state)
4 43
 
5 44
 
6 45
 def named_product(**items):
Browse code

move adaptive/learner/utils.py to adaptive/utils.py

Bas Nijholt authored on 13/06/2018 01:33:03
Showing 1 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,19 @@
1
+# -*- coding: utf-8 -*-
2
+from contextlib import contextmanager
3
+from itertools import product
4
+
5
+
6
+def named_product(**items):
7
+    names = items.keys()
8
+    vals = items.values()
9
+    return [dict(zip(names, res)) for res in product(*vals)]
10
+
11
+
12
+@contextmanager
13
+def restore(*learners):
14
+    states = [learner.__getstate__() for learner in learners]
15
+    try:
16
+        yield
17
+    finally:
18
+        for state, learner in zip(states, learners):
19
+            learner.__setstate__(state)