Mercurial > templog
annotate web/log.py @ 153:0529f3527626
config for sensor IDs
author | Matt Johnston <matt@ucc.asn.au> |
---|---|
date | Thu, 20 Dec 2012 22:26:18 +0800 |
parents | 3b4277aaed3c |
children | 683cfd134f6a |
rev | line source |
---|---|
78 | 1 # -*- coding: utf-8 -*- |
37
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
2 #:vim:et:ts=4:sts=4:sw=4: |
27 | 3 import rrdtool |
4 import os | |
29 | 5 import os.path |
27 | 6 import sys |
29 | 7 import glob |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
8 import hashlib |
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
9 import tempfile |
37
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
10 import time |
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
11 import syslog |
44 | 12 import sqlite3 |
49
206294a354a7
update values one at a time, ignore failure
Matt Johnston <matt@ucc.asn.au>
parents:
45
diff
changeset
|
13 import traceback |
67 | 14 import datetime |
75
ca08442635ca
report raw ds18b20 values instead
Matt Johnston <matt@ucc.asn.au>
parents:
73
diff
changeset
|
15 import struct |
77 | 16 import binascii |
29 | 17 from colorsys import hls_to_rgb |
27 | 18 |
28 | 19 import config |
27 | 20 |
21 def sensor_rrd_path(s): | |
28 | 22 return '%s/sensor_%s.rrd' % (config.DATA_PATH, s) |
27 | 23 |
29 | 24 # returns (path, sensor_name) tuples |
25 def all_sensors(): | |
26 return [(r, os.path.basename(r[:-4])) | |
27 for r in glob.glob('%s/*.rrd' % config.DATA_PATH)] | |
28 | |
27 | 29 def create_rrd(sensor_id): |
43
ea99aae87884
Create with a start date so that it works
Matt Johnston <matt@ucc.asn.au>
parents:
40
diff
changeset
|
30 # start date of 10 seconds into 1970 is used so that we can |
ea99aae87884
Create with a start date so that it works
Matt Johnston <matt@ucc.asn.au>
parents:
40
diff
changeset
|
31 # update with prior values straight away. |
61 | 32 if 'voltage' in sensor_id: |
63 | 33 args = [ |
34 '--step', '3600', | |
35 'DS:temp:GAUGE:7200:1:10', | |
36 'RRA:AVERAGE:0.5:1:87600'] | |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
37 elif 'fridge_on' in sensor_id: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
38 args = [ |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
39 '--step', '300', |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
40 'DS:temp:GAUGE:600:-100:500', |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
41 'RRA:LAST:0.5:1:1051200'] |
61 | 42 else: |
63 | 43 args = [ |
44 '--step', '300', | |
88
6f4497a448e8
increase temperature logging range
Matt Johnston <matt@ucc.asn.au>
parents:
78
diff
changeset
|
45 'DS:temp:GAUGE:600:-100:500', |
61 | 46 'RRA:AVERAGE:0.5:1:1051200'] |
27 | 47 |
61 | 48 rrdtool.create(sensor_rrd_path(sensor_id), |
63 | 49 '--start', 'now-60d', |
61 | 50 *args) |
29 | 51 |
52 # stolen from viewmtn, stolen from monotone-viz | |
53 def colour_from_string(str): | |
54 def f(off): | |
55 return ord(hashval[off]) / 256.0 | |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
56 hashval = hashlib.sha1(str).digest() |
29 | 57 hue = f(5) |
58 li = f(1) * 0.15 + 0.55 | |
59 sat = f(2) * 0.5 + .5 | |
60 return ''.join(["%.2x" % int(x * 256) for x in hls_to_rgb(hue, li, sat)]) | |
61 | |
62 def graph_png(start, length): | |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
63 os.environ['MATT_PNG_BODGE_COMPRESS'] = '4' |
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
64 os.environ['MATT_PNG_BODGE_FILTER'] = 'paeth' |
29 | 65 rrds = all_sensors() |
66 | |
67 graph_args = [] | |
61 | 68 have_volts = False |
29 | 69 for n, (rrdfile, sensor) in enumerate(rrds): |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
70 unit = None |
61 | 71 if 'avrtemp' in sensor: |
72 continue | |
73 if 'voltage' in sensor: | |
74 have_volts = True | |
75 vname = 'scalevolts' | |
71
7d243ba2dd39
swap the left and right scales, kind of clunky
Matt Johnston <matt@ucc.asn.au>
parents:
69
diff
changeset
|
76 graph_args.append('DEF:%(vname)s=%(rrdfile)s:temp:AVERAGE:step=3600' % locals()) |
78 | 77 unit = 'V' |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
78 elif 'fridge_on' in sensor: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
79 vname = 'fridge_on' |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
80 graph_args.append('DEF:raw%(vname)s=%(rrdfile)s:temp:LAST' % locals()) |
131 | 81 graph_args.append('CDEF:%(vname)s=raw%(vname)s,-0.2,*,3,+' % locals()) |
61 | 82 else: |
83 vname = 'temp%d' % n | |
71
7d243ba2dd39
swap the left and right scales, kind of clunky
Matt Johnston <matt@ucc.asn.au>
parents:
69
diff
changeset
|
84 graph_args.append('DEF:raw%(vname)s=%(rrdfile)s:temp:AVERAGE' % locals()) |
101
fd8482365489
limit temps to 35º on graph
Matt Johnston <matt@ucc.asn.au>
parents:
99
diff
changeset
|
85 # limit max temp to 50 |
fd8482365489
limit temps to 35º on graph
Matt Johnston <matt@ucc.asn.au>
parents:
99
diff
changeset
|
86 graph_args.append('CDEF:%(vname)s=raw%(vname)s,35,GT,UNKN,raw%(vname)s,0.1,*,2,+,IF' % locals()) |
78 | 87 unit = '<span face="Liberation Serif">º</span>C' |
88 | |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
89 format_last_value = None |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
90 if unit: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
91 try: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
92 last_value = float(rrdtool.info(rrdfile)['ds[temp].last_ds']) |
137
d36fe81077cc
strip multiple-of-ten temperatures correctly
Matt Johnston <matt@ucc.asn.au>
parents:
131
diff
changeset
|
93 format_last_value = ('%f' % last_value).rstrip('0').rstrip('.') + unit |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
94 except ValueError: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
95 pass |
29 | 96 width = config.LINE_WIDTH |
97 legend = config.SENSOR_NAMES.get(sensor, sensor) | |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
98 colour = config.SENSOR_COLOURS.get(legend, colour_from_string(sensor)) |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
99 if format_last_value: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
100 print_legend = '%s (%s)' % (legend, format_last_value) |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
101 else: |
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
102 print_legend = legend |
78 | 103 graph_args.append('LINE%(width)f:%(vname)s#%(colour)s:%(print_legend)s' % locals()) |
29 | 104 |
67 | 105 end = int(start+length) |
106 start = int(start) | |
107 | |
29 | 108 tempf = tempfile.NamedTemporaryFile() |
67 | 109 dateformat = '%H:%M:%S %Y-%m-%d' |
110 watermark = ("Now %s\t" | |
111 "Start %s\t" | |
112 "End %s" % ( | |
113 datetime.datetime.now().strftime(dateformat), | |
114 datetime.datetime.fromtimestamp(start).strftime(dateformat), | |
115 datetime.datetime.fromtimestamp(end).strftime(dateformat) )) | |
116 | |
117 args = [tempf.name, '-s', str(start), | |
118 '-e', str(end), | |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
119 '-w', str(config.GRAPH_WIDTH), |
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
120 '-h', str(config.GRAPH_HEIGHT), |
29 | 121 '--slope-mode', |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
122 '--border', '0', |
73
0a8639039453
Get rid of axes labels, tidy html
Matt Johnston <matt@ucc.asn.au>
parents:
71
diff
changeset
|
123 # '--vertical-label', 'Voltage', |
71
7d243ba2dd39
swap the left and right scales, kind of clunky
Matt Johnston <matt@ucc.asn.au>
parents:
69
diff
changeset
|
124 '--y-grid', '0.1:1', |
7d243ba2dd39
swap the left and right scales, kind of clunky
Matt Johnston <matt@ucc.asn.au>
parents:
69
diff
changeset
|
125 '--dynamic-labels', |
56 | 126 '--grid-dash', '1:0', |
77 | 127 '--zoom', str(config.ZOOM), |
56 | 128 '--color', 'GRID#00000000', |
129 '--color', 'MGRID#aaaaaa', | |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
130 '--color', 'BACK#ffffff', |
67 | 131 '--disable-rrdtool-tag', |
78 | 132 '--pango-markup', |
67 | 133 '--watermark', watermark, |
31
5e75e08d20ac
- Various fixes for web server, kind of works
Matt Johnston <matt@ucc.asn.au>
parents:
29
diff
changeset
|
134 '--imgformat', 'PNG'] \ |
29 | 135 + graph_args |
67 | 136 args += ['--font', 'DEFAULT:12:%s' % config.GRAPH_FONT] |
137 args += ['--font', 'WATERMARK:10:%s' % config.GRAPH_FONT] | |
61 | 138 if have_volts: |
71
7d243ba2dd39
swap the left and right scales, kind of clunky
Matt Johnston <matt@ucc.asn.au>
parents:
69
diff
changeset
|
139 args += ['--right-axis', '10:-20', # matches the scalevolts CDEF above |
73
0a8639039453
Get rid of axes labels, tidy html
Matt Johnston <matt@ucc.asn.au>
parents:
71
diff
changeset
|
140 '--right-axis-format', '%.0lf', |
0a8639039453
Get rid of axes labels, tidy html
Matt Johnston <matt@ucc.asn.au>
parents:
71
diff
changeset
|
141 # '--right-axis-label', 'Temperature' |
0a8639039453
Get rid of axes labels, tidy html
Matt Johnston <matt@ucc.asn.au>
parents:
71
diff
changeset
|
142 ] |
61 | 143 |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
144 print>>sys.stderr, ' '.join("'%s'" % s for s in args) |
29 | 145 rrdtool.graph(*args) |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
146 #return tempf |
29 | 147 return tempf.read() |
148 | |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
149 def validate_value(m): |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
150 if m == 85: |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
151 yield 'U' |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
152 else: |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
153 yield '%f' % m |
125 | 154 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
155 def sensor_update(sensor_id, measurements): |
27 | 156 try: |
157 open(sensor_rrd_path(sensor_id)) | |
158 except IOError, e: | |
159 create_rrd(sensor_id) | |
160 | |
37
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
161 if measurements: |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
162 values = ['%d:%s' % (t, validate_value(m)) for (t, m) in measurements] |
27 | 163 |
37
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
164 rrdfile = sensor_rrd_path(sensor_id) |
40 | 165 # XXX what to do here when it fails... |
49
206294a354a7
update values one at a time, ignore failure
Matt Johnston <matt@ucc.asn.au>
parents:
45
diff
changeset
|
166 for v in values: |
206294a354a7
update values one at a time, ignore failure
Matt Johnston <matt@ucc.asn.au>
parents:
45
diff
changeset
|
167 try: |
206294a354a7
update values one at a time, ignore failure
Matt Johnston <matt@ucc.asn.au>
parents:
45
diff
changeset
|
168 rrdtool.update(rrdfile, v) |
63 | 169 except rrdtool.error, e: |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
170 print>>sys.stderr, "Bad rrdtool update '%s': %s" % (v, str(e)) |
49
206294a354a7
update values one at a time, ignore failure
Matt Johnston <matt@ucc.asn.au>
parents:
45
diff
changeset
|
171 traceback.print_exc(file=sys.stderr) |
33 | 172 |
37
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
173 # be paranoid |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
174 #f = file(rrdfile) |
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
175 #os.fsync(f.fileno()) |
27 | 176 |
103 | 177 def debug_file(mode='r'): |
178 return open('%s/debug.log' % config.DATA_PATH, mode) | |
179 | |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
180 def record_debug(params): |
103 | 181 f = debug_file('a+') |
37
8da0fdadc8d7
- Getting there, update has problems
Matt Johnston <matt@ucc.asn.au>
parents:
33
diff
changeset
|
182 f.write('===== %s =====\n' % time.strftime('%a, %d %b %Y %H:%M:%S')) |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
183 json.dump(params, f, sort_keys=True, indent=4)) |
33 | 184 f.flush() |
185 return f | |
186 | |
103 | 187 |
188 def tail_debug_log(): | |
189 f = debug_file() | |
190 f.seek(0, 2) | |
191 size = f.tell() | |
192 f.seek(max(0, size-30000)) | |
193 return '\n'.join(l.strip() for l in f.readlines()[-400:]) | |
194 | |
75
ca08442635ca
report raw ds18b20 values instead
Matt Johnston <matt@ucc.asn.au>
parents:
73
diff
changeset
|
195 def convert_ds18b20_12bit(reading): |
ca08442635ca
report raw ds18b20 values instead
Matt Johnston <matt@ucc.asn.au>
parents:
73
diff
changeset
|
196 value = struct.unpack('>h', binascii.unhexlify(reading))[0] |
ca08442635ca
report raw ds18b20 values instead
Matt Johnston <matt@ucc.asn.au>
parents:
73
diff
changeset
|
197 return value * 0.0625 |
ca08442635ca
report raw ds18b20 values instead
Matt Johnston <matt@ucc.asn.au>
parents:
73
diff
changeset
|
198 |
94 | 199 def time_rem(name, entries): |
200 val_ticks = int(entries[name]) | |
95 | 201 val_rem = int(entries['%s_rem' % name]) |
99
1a88bb989afb
fix off-by-one in remainder code
Matt Johnston <matt@ucc.asn.au>
parents:
95
diff
changeset
|
202 tick_wake = int(entries['tick_wake']) + 1 |
94 | 203 tick_secs = int(entries['tick_secs']) |
204 return val_ticks + float(val_rem) * tick_secs / tick_wake | |
205 | |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
206 def parse(params): |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
207 |
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
208 start_time = time.time() |
33 | 209 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
210 debugf = record_debug(params) |
27 | 211 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
212 remote_now = params['now'] |
27 | 213 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
214 time_diff = start_time - remote_now |
27 | 215 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
216 # readings is [ ({sensorname: value, ...}, time), ... ] |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
217 readings = params['readings'] |
27 | 218 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
219 # measurements is {sensorname: [(time, value), ...], ...} |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
220 measurements = {} |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
221 for rs, t in readings: |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
222 real_t = t + time_diff |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
223 for s, v in rs.iteritems(): |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
224 measurements.getdefault(s, []).append((real_t, v)) |
61 | 225 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
226 # one-off measurements here |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
227 measurements['fridge_on'] = [ (time.time(), params['fridge_on']) ] |
126
6a9419ac8f77
ui tweaks, add fridge values
Matt Johnston <matt@ucc.asn.au>
parents:
125
diff
changeset
|
228 |
146
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
229 for s, vs in measurements.iteritems(): |
3b4277aaed3c
update web to handle new style params
Matt Johnston <matt@ucc.asn.au>
parents:
137
diff
changeset
|
230 sensor_update(s, vs) |
33 | 231 |
104
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
232 timedelta = time.time() - start_time |
7f3fc0980df1
- few more web tweaks. don't fsync, it's slow.
Matt Johnston <matt@ucc.asn.au>
parents:
103
diff
changeset
|
233 debugf.write("Updated %d sensors in %.2f secs\n" % (len(sensors), timedelta)) |
33 | 234 debugf.flush() |