... | ... |
@@ -76,10 +76,12 @@ class _RequireAttrsABCMeta(abc.ABCMeta): |
76 | 76 |
for name, type_ in obj.__annotations__.items(): |
77 | 77 |
try: |
78 | 78 |
x = getattr(obj, name) |
79 |
- if not isinstance(x, type_): |
|
80 |
- raise TypeError( |
|
81 |
- f"The attribute '{name}' is of {type_} instead of {type(x)}." |
|
82 |
- ) |
|
83 | 79 |
except AttributeError: |
84 |
- raise AttributeError(f"Required attribute {name} not set in __init__.") |
|
80 |
+ raise AttributeError( |
|
81 |
+ f"Required attribute {name} not set in __init__." |
|
82 |
+ ) from None |
|
83 |
+ else: |
|
84 |
+ if not isinstance(x, type_): |
|
85 |
+ msg = f"The attribute '{name}' should be of type {type_}, not {type(x)}." |
|
86 |
+ raise TypeError(msg) |
|
85 | 87 |
return obj |
... | ... |
@@ -80,6 +80,6 @@ class _RequireAttrsABCMeta(abc.ABCMeta): |
80 | 80 |
raise TypeError( |
81 | 81 |
f"The attribute '{name}' is of {type_} instead of {type(x)}." |
82 | 82 |
) |
83 |
- except AttributeError as e: |
|
84 |
- raise e(f"Required attribute {name} not set in __init__.") |
|
83 |
+ except AttributeError: |
|
84 |
+ raise AttributeError(f"Required attribute {name} not set in __init__.") |
|
85 | 85 |
return obj |
... | ... |
@@ -74,11 +74,12 @@ class _RequireAttrsABCMeta(abc.ABCMeta): |
74 | 74 |
def __call__(self, *args, **kwargs): |
75 | 75 |
obj = super().__call__(*args, **kwargs) |
76 | 76 |
for name, type_ in obj.__annotations__.items(): |
77 |
- if not hasattr(obj, name): |
|
78 |
- raise ValueError(f"Required attribute {name} not set in __init__.") |
|
79 |
- elif not isinstance(getattr(obj, name), type_): |
|
80 |
- raise TypeError( |
|
81 |
- f"The attribute '{name}' is of {type_} instead of {type(getattr(obj, name))}" |
|
82 |
- ) |
|
83 |
- |
|
77 |
+ try: |
|
78 |
+ x = getattr(obj, name) |
|
79 |
+ if not isinstance(x, type_): |
|
80 |
+ raise TypeError( |
|
81 |
+ f"The attribute '{name}' is of {type_} instead of {type(x)}." |
|
82 |
+ ) |
|
83 |
+ except AttributeError as e: |
|
84 |
+ raise e(f"Required attribute {name} not set in __init__.") |
|
84 | 85 |
return obj |
... | ... |
@@ -71,11 +71,14 @@ def copy_docstring_from(other): |
71 | 71 |
|
72 | 72 |
|
73 | 73 |
class _RequireAttrsABCMeta(abc.ABCMeta): |
74 |
- required_attributes = [] |
|
75 |
- |
|
76 | 74 |
def __call__(self, *args, **kwargs): |
77 | 75 |
obj = super().__call__(*args, **kwargs) |
78 |
- for name in obj.required_attributes: |
|
76 |
+ for name, type_ in obj.__annotations__.items(): |
|
79 | 77 |
if not hasattr(obj, name): |
80 | 78 |
raise ValueError(f"Required attribute {name} not set in __init__.") |
79 |
+ elif not isinstance(getattr(obj, name), type_): |
|
80 |
+ raise TypeError( |
|
81 |
+ f"The attribute '{name}' is of {type_} instead of {type(getattr(obj, name))}" |
|
82 |
+ ) |
|
83 |
+ |
|
81 | 84 |
return obj |
... | ... |
@@ -1,5 +1,6 @@ |
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 |
|
3 |
+import abc |
|
3 | 4 |
import functools |
4 | 5 |
import gzip |
5 | 6 |
import os |
... | ... |
@@ -67,3 +68,14 @@ def copy_docstring_from(other): |
67 | 68 |
return functools.wraps(other)(method) |
68 | 69 |
|
69 | 70 |
return decorator |
71 |
+ |
|
72 |
+ |
|
73 |
+class _RequireAttrsABCMeta(abc.ABCMeta): |
|
74 |
+ required_attributes = [] |
|
75 |
+ |
|
76 |
+ def __call__(self, *args, **kwargs): |
|
77 |
+ obj = super().__call__(*args, **kwargs) |
|
78 |
+ for name in obj.required_attributes: |
|
79 |
+ if not hasattr(obj, name): |
|
80 |
+ raise ValueError(f"Required attribute {name} not set in __init__.") |
|
81 |
+ return obj |
Right now, if a program crashes in the middle of saving, you lose
all your data.
Right now, if a program crashes in the middle of saving, you lose
all your data. This ensures that the old file is first moved, then
the new file is saved, and only then the old file is removed.
... | ... |
@@ -7,6 +7,8 @@ import pickle |
7 | 7 |
from contextlib import contextmanager |
8 | 8 |
from itertools import product |
9 | 9 |
|
10 |
+from atomicwrites import AtomicWriter |
|
11 |
+ |
|
10 | 12 |
|
11 | 13 |
def named_product(**items): |
12 | 14 |
names = items.keys() |
... | ... |
@@ -44,9 +46,13 @@ def save(fname, data, compress=True): |
44 | 46 |
dirname = os.path.dirname(fname) |
45 | 47 |
if dirname: |
46 | 48 |
os.makedirs(dirname, exist_ok=True) |
47 |
- _open = gzip.open if compress else open |
|
48 |
- with _open(fname, "wb") as f: |
|
49 |
- pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) |
|
49 |
+ |
|
50 |
+ blob = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL) |
|
51 |
+ if compress: |
|
52 |
+ blob = gzip.compress(blob) |
|
53 |
+ |
|
54 |
+ with AtomicWriter(fname, "wb", overwrite=True).open() as f: |
|
55 |
+ f.write(blob) |
|
50 | 56 |
|
51 | 57 |
|
52 | 58 |
def load(fname, compress=True): |
... | ... |
@@ -4,7 +4,6 @@ import functools |
4 | 4 |
import gzip |
5 | 5 |
import os |
6 | 6 |
import pickle |
7 |
-import time |
|
8 | 7 |
from contextlib import contextmanager |
9 | 8 |
from itertools import product |
10 | 9 |
|
... | ... |
@@ -28,13 +27,15 @@ def restore(*learners): |
28 | 27 |
def cache_latest(f): |
29 | 28 |
"""Cache the latest return value of the function and add it |
30 | 29 |
as 'self._cache[f.__name__]'.""" |
30 |
+ |
|
31 | 31 |
@functools.wraps(f) |
32 | 32 |
def wrapper(*args, **kwargs): |
33 | 33 |
self = args[0] |
34 |
- if not hasattr(self, '_cache'): |
|
34 |
+ if not hasattr(self, "_cache"): |
|
35 | 35 |
self._cache = {} |
36 | 36 |
self._cache[f.__name__] = f(*args, **kwargs) |
37 | 37 |
return self._cache[f.__name__] |
38 |
+ |
|
38 | 39 |
return wrapper |
39 | 40 |
|
40 | 41 |
|
... | ... |
@@ -44,18 +45,19 @@ def save(fname, data, compress=True): |
44 | 45 |
if dirname: |
45 | 46 |
os.makedirs(dirname, exist_ok=True) |
46 | 47 |
_open = gzip.open if compress else open |
47 |
- with _open(fname, 'wb') as f: |
|
48 |
+ with _open(fname, "wb") as f: |
|
48 | 49 |
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) |
49 | 50 |
|
50 | 51 |
|
51 | 52 |
def load(fname, compress=True): |
52 | 53 |
fname = os.path.expanduser(fname) |
53 | 54 |
_open = gzip.open if compress else open |
54 |
- with _open(fname, 'rb') as f: |
|
55 |
+ with _open(fname, "rb") as f: |
|
55 | 56 |
return pickle.load(f) |
56 | 57 |
|
57 | 58 |
|
58 | 59 |
def copy_docstring_from(other): |
59 | 60 |
def decorator(method): |
60 | 61 |
return functools.wraps(other)(method) |
62 |
+ |
|
61 | 63 |
return decorator |
... | ... |
@@ -1,12 +1,12 @@ |
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 |
|
3 |
-from contextlib import contextmanager |
|
4 | 3 |
import functools |
5 | 4 |
import gzip |
6 |
-from itertools import product |
|
7 | 5 |
import os |
8 | 6 |
import pickle |
9 | 7 |
import time |
8 |
+from contextlib import contextmanager |
|
9 |
+from itertools import product |
|
10 | 10 |
|
11 | 11 |
|
12 | 12 |
def named_product(**items): |
See https://www.python.org/dev/peps/pep-0008/#imports
Bas Nijholt authored on 26/11/2018 13:39:12Previously we were wrapping the learner function and doing the
timing there. This makes things more complicated in the case that
we are learning an 'async def' function.
... | ... |
@@ -8,12 +8,6 @@ import pickle |
8 | 8 |
import time |
9 | 9 |
|
10 | 10 |
|
11 |
-def timed(f, *args, **kwargs): |
|
12 |
- t_start = time.time() |
|
13 |
- result = f(*args, **kwargs) |
|
14 |
- return result, time.time() - t_start |
|
15 |
- |
|
16 |
- |
|
17 | 11 |
def named_product(**items): |
18 | 12 |
names = items.keys() |
19 | 13 |
vals = items.values() |
... | ... |
@@ -1,7 +1,10 @@ |
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 |
from contextlib import contextmanager |
3 |
-from functools import wraps |
|
3 |
+import functools |
|
4 |
+import gzip |
|
4 | 5 |
from itertools import product |
6 |
+import os |
|
7 |
+import pickle |
|
5 | 8 |
import time |
6 | 9 |
|
7 | 10 |
|
... | ... |
@@ -30,7 +33,7 @@ def restore(*learners): |
30 | 33 |
def cache_latest(f): |
31 | 34 |
"""Cache the latest return value of the function and add it |
32 | 35 |
as 'self._cache[f.__name__]'.""" |
33 |
- @wraps(f) |
|
36 |
+ @functools.wraps(f) |
|
34 | 37 |
def wrapper(*args, **kwargs): |
35 | 38 |
self = args[0] |
36 | 39 |
if not hasattr(self, '_cache'): |
... | ... |
@@ -38,3 +41,26 @@ def cache_latest(f): |
38 | 41 |
self._cache[f.__name__] = f(*args, **kwargs) |
39 | 42 |
return self._cache[f.__name__] |
40 | 43 |
return wrapper |
44 |
+ |
|
45 |
+ |
|
46 |
+def save(fname, data, compress=True): |
|
47 |
+ fname = os.path.expanduser(fname) |
|
48 |
+ dirname = os.path.dirname(fname) |
|
49 |
+ if dirname: |
|
50 |
+ os.makedirs(dirname, exist_ok=True) |
|
51 |
+ _open = gzip.open if compress else open |
|
52 |
+ with _open(fname, 'wb') as f: |
|
53 |
+ pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) |
|
54 |
+ |
|
55 |
+ |
|
56 |
+def load(fname, compress=True): |
|
57 |
+ fname = os.path.expanduser(fname) |
|
58 |
+ _open = gzip.open if compress else open |
|
59 |
+ with _open(fname, 'rb') as f: |
|
60 |
+ return pickle.load(f) |
|
61 |
+ |
|
62 |
+ |
|
63 |
+def copy_docstring_from(other): |
|
64 |
+ def decorator(method): |
|
65 |
+ return functools.wraps(other)(method) |
|
66 |
+ return decorator |
... | ... |
@@ -1,5 +1,6 @@ |
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 |
from contextlib import contextmanager |
3 |
+from functools import wraps |
|
3 | 4 |
from itertools import product |
4 | 5 |
import time |
5 | 6 |
|
... | ... |
@@ -24,3 +25,16 @@ def restore(*learners): |
24 | 25 |
finally: |
25 | 26 |
for state, learner in zip(states, learners): |
26 | 27 |
learner.__setstate__(state) |
28 |
+ |
|
29 |
+ |
|
30 |
+def cache_latest(f): |
|
31 |
+ """Cache the latest return value of the function and add it |
|
32 |
+ as 'self._cache[f.__name__]'.""" |
|
33 |
+ @wraps(f) |
|
34 |
+ def wrapper(*args, **kwargs): |
|
35 |
+ self = args[0] |
|
36 |
+ if not hasattr(self, '_cache'): |
|
37 |
+ self._cache = {} |
|
38 |
+ self._cache[f.__name__] = f(*args, **kwargs) |
|
39 |
+ return self._cache[f.__name__] |
|
40 |
+ return wrapper |
... | ... |
@@ -4,14 +4,10 @@ from itertools import product |
4 | 4 |
import time |
5 | 5 |
|
6 | 6 |
|
7 |
-class TimeReturn: |
|
8 |
- def __init__(self, function): |
|
9 |
- self.function = function |
|
10 |
- |
|
11 |
- def __call__(self, *args, **kwargs): |
|
12 |
- t_start = time.time() |
|
13 |
- result = self.function(*args, **kwargs) |
|
14 |
- return result, time.time() - t_start |
|
7 |
+def timed(f, *args, **kwargs): |
|
8 |
+ t_start = time.time() |
|
9 |
+ result = f(*args, **kwargs) |
|
10 |
+ return result, time.time() - t_start |
|
15 | 11 |
|
16 | 12 |
|
17 | 13 |
def named_product(**items): |
... | ... |
@@ -5,21 +5,13 @@ import time |
5 | 5 |
|
6 | 6 |
|
7 | 7 |
class TimeReturn: |
8 |
- def __init__(self, function, total_time=0): |
|
8 |
+ def __init__(self, function): |
|
9 | 9 |
self.function = function |
10 |
- self.total_time = total_time |
|
11 | 10 |
|
12 | 11 |
def __call__(self, *args, **kwargs): |
13 | 12 |
t_start = time.time() |
14 | 13 |
result = self.function(*args, **kwargs) |
15 |
- self.total_time += time.time() - t_start |
|
16 |
- return result, self.total_time |
|
17 |
- |
|
18 |
- def __getstate__(self): |
|
19 |
- return (self.function, self.total_time) |
|
20 |
- |
|
21 |
- def __setstate__(self, state): |
|
22 |
- self.__init__(*state) |
|
14 |
+ return result, time.time() - t_start |
|
23 | 15 |
|
24 | 16 |
|
25 | 17 |
def named_product(**items): |
... | ... |
@@ -4,24 +4,6 @@ from itertools import product |
4 | 4 |
import time |
5 | 5 |
|
6 | 6 |
|
7 |
-class WithTime: |
|
8 |
- def __init__(self, function, time=0): |
|
9 |
- self.function = function |
|
10 |
- self.time = time |
|
11 |
- |
|
12 |
- def __call__(self, *args, **kwargs): |
|
13 |
- t_start = time.time() |
|
14 |
- result = self.function(*args, **kwargs) |
|
15 |
- self.time += time.time() - t_start |
|
16 |
- return result |
|
17 |
- |
|
18 |
- def __getstate__(self): |
|
19 |
- return (self.function, self.time) |
|
20 |
- |
|
21 |
- def __setstate__(self, state): |
|
22 |
- self.__init__(*state) |
|
23 |
- |
|
24 |
- |
|
25 | 7 |
class TimeReturn: |
26 | 8 |
def __init__(self, function, total_time=0): |
27 | 9 |
self.function = function |
... | ... |
@@ -22,26 +22,6 @@ class WithTime: |
22 | 22 |
self.__init__(*state) |
23 | 23 |
|
24 | 24 |
|
25 |
-class AverageTimeReturn: |
|
26 |
- def __init__(self, function, total_time=0, n=0): |
|
27 |
- self.function = function |
|
28 |
- self.total_time = total_time |
|
29 |
- self.n = n |
|
30 |
- |
|
31 |
- def __call__(self, *args, **kwargs): |
|
32 |
- t_start = time.time() |
|
33 |
- result = self.function(*args, **kwargs) |
|
34 |
- self.total_time += time.time() - t_start |
|
35 |
- self.n += 1 |
|
36 |
- return result, self.total_time / self.n |
|
37 |
- |
|
38 |
- def __getstate__(self): |
|
39 |
- return (self.function, self.total_time, self.n) |
|
40 |
- |
|
41 |
- def __setstate__(self, state): |
|
42 |
- self.__init__(*state) |
|
43 |
- |
|
44 |
- |
|
45 | 25 |
class TimeReturn: |
46 | 26 |
def __init__(self, function, total_time=0): |
47 | 27 |
self.function = function |
... | ... |
@@ -42,6 +42,24 @@ class AverageTimeReturn: |
42 | 42 |
self.__init__(*state) |
43 | 43 |
|
44 | 44 |
|
45 |
+class TimeReturn: |
|
46 |
+ def __init__(self, function, total_time=0): |
|
47 |
+ self.function = function |
|
48 |
+ self.total_time = total_time |
|
49 |
+ |
|
50 |
+ def __call__(self, *args, **kwargs): |
|
51 |
+ t_start = time.time() |
|
52 |
+ result = self.function(*args, **kwargs) |
|
53 |
+ self.total_time += time.time() - t_start |
|
54 |
+ return result, self.total_time |
|
55 |
+ |
|
56 |
+ def __getstate__(self): |
|
57 |
+ return (self.function, self.total_time) |
|
58 |
+ |
|
59 |
+ def __setstate__(self, state): |
|
60 |
+ self.__init__(*state) |
|
61 |
+ |
|
62 |
+ |
|
45 | 63 |
def named_product(**items): |
46 | 64 |
names = items.keys() |
47 | 65 |
vals = items.values() |
... | ... |
@@ -1,6 +1,45 @@ |
1 | 1 |
# -*- coding: utf-8 -*- |
2 | 2 |
from contextlib import contextmanager |
3 | 3 |
from itertools import product |
4 |
+import time |
|
5 |
+ |
|
6 |
+ |
|
7 |
+class WithTime: |
|
8 |
+ def __init__(self, function, time=0): |
|
9 |
+ self.function = function |
|
10 |
+ self.time = time |
|
11 |
+ |
|
12 |
+ def __call__(self, *args, **kwargs): |
|
13 |
+ t_start = time.time() |
|
14 |
+ result = self.function(*args, **kwargs) |
|
15 |
+ self.time += time.time() - t_start |
|
16 |
+ return result |
|
17 |
+ |
|
18 |
+ def __getstate__(self): |
|
19 |
+ return (self.function, self.time) |
|
20 |
+ |
|
21 |
+ def __setstate__(self, state): |
|
22 |
+ self.__init__(*state) |
|
23 |
+ |
|
24 |
+ |
|
25 |
+class AverageTimeReturn: |
|
26 |
+ def __init__(self, function, total_time=0, n=0): |
|
27 |
+ self.function = function |
|
28 |
+ self.total_time = total_time |
|
29 |
+ self.n = n |
|
30 |
+ |
|
31 |
+ def __call__(self, *args, **kwargs): |
|
32 |
+ t_start = time.time() |
|
33 |
+ result = self.function(*args, **kwargs) |
|
34 |
+ self.total_time += time.time() - t_start |
|
35 |
+ self.n += 1 |
|
36 |
+ return result, self.total_time / self.n |
|
37 |
+ |
|
38 |
+ def __getstate__(self): |
|
39 |
+ return (self.function, self.total_time, self.n) |
|
40 |
+ |
|
41 |
+ def __setstate__(self, state): |
|
42 |
+ self.__init__(*state) |
|
4 | 43 |
|
5 | 44 |
|
6 | 45 |
def named_product(**items): |
1 | 1 |
new file mode 100644 |
... | ... |
@@ -0,0 +1,19 @@ |
1 |
+# -*- coding: utf-8 -*- |
|
2 |
+from contextlib import contextmanager |
|
3 |
+from itertools import product |
|
4 |
+ |
|
5 |
+ |
|
6 |
+def named_product(**items): |
|
7 |
+ names = items.keys() |
|
8 |
+ vals = items.values() |
|
9 |
+ return [dict(zip(names, res)) for res in product(*vals)] |
|
10 |
+ |
|
11 |
+ |
|
12 |
+@contextmanager |
|
13 |
+def restore(*learners): |
|
14 |
+ states = [learner.__getstate__() for learner in learners] |
|
15 |
+ try: |
|
16 |
+ yield |
|
17 |
+ finally: |
|
18 |
+ for state, learner in zip(states, learners): |
|
19 |
+ learner.__setstate__(state) |