-
-
Notifications
You must be signed in to change notification settings - Fork 14
/
Copy pathcollect.py
340 lines (278 loc) · 14.9 KB
/
collect.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
from utils import *
import subprocess
import copy
from project import Project, SharedProject
from codegen.caravel_codegen import generate_openlane_files, generate_sby_file, json_config
from codegen.allocator import allocate_macros
from urllib.parse import urlparse
REQUIRED_KEYS_GROUP = ["interfaces", "openram_support", "configuration", "docs", "projects"]
REQUIRED_KEYS_LOCAL = ["project_directory", "caravel", "env"]
def merge_two_dicts(x, y):
z = x.copy() # start with keys and values of x
z.update(y) # modifies z with keys and values of y
return z
class Collection(object):
def __init__(self, args):
self.args = args
project_config = parse_config(args.config, REQUIRED_KEYS_GROUP)
try:
local_config = parse_config(args.local_config, REQUIRED_KEYS_LOCAL)
except FileNotFoundError:
logging.error("%s local config not found. Copy matt_local.yaml to local.yaml and adjust as necessary" % args.local_config)
exit(1)
self.config = merge_two_dicts(project_config, local_config)
self.projects = []
if not (0 < len(self.config['projects']) <= 16):
logging.error("bad number of projects - must be > 0 and <= 16")
exit(1)
# build the list of projects
for project_info in self.config['projects'].values():
repo = project_info["repo"]
try:
directory = project_info["dir"]
except KeyError:
# the project's directory is made by joining project dir to last part of the repo url
parsed = urlparse(repo)
directory = parsed.path.rpartition('/')[-1]
directory = os.path.join(self.config['project_directory'], directory)
commit = project_info["commit"]
pos = project_info["pos"]
required_interfaces = list(self.config['interfaces']['required'].keys())
project = Project(args, repo, directory, commit, pos, required_interfaces, self.config)
# if --project is given, skip others
if self.args.project is not None:
if self.args.project != project.id:
continue
# start from a given project if --from is given
if self.args.test_from is not None:
if project.id < self.args.test_from:
continue
# append
self.projects.append(project)
self.shared_projects = []
if self.args.openram:
for project_info in self.config['openram_support']['projects'].values():
directory = None
repo = project_info["repo"]
try:
directory = project_info["dir"]
except KeyError:
# the project's directory is made by joining project dir to last part of the repo url
parsed = urlparse(repo)
directory = parsed.path.rpartition('/')[-1]
directory = os.path.join(self.config['project_directory'], directory)
commit = project_info["commit"]
pos = project_info["pos"]
project = SharedProject(args, repo, directory, commit, pos, self.config)
self.shared_projects.append(project)
logging.info(project)
# fill space with duplicated projects
if args.fill and args.fill > len(self.projects):
num_real_projects = len(self.projects)
# make the copies
for i in range(len(self.projects), args.fill):
dup_project = copy.deepcopy(self.projects[i % num_real_projects])
dup_project.id = i
# instance names are generated now
# dup_project.config['caravel_test']['instance_name'] += str(dup_project.id)
self.projects.append(dup_project)
# assert ids are unique
ids = [project.id for project in self.projects]
if len(ids) != len(set(ids)):
logging.error("not all project ids are unique: %s" % ids)
exit(1)
self.macro_allocation = {}
self.width = self.config['configuration']['user_area_width']
self.height = self.config['configuration']['user_area_height']
self.interface_definitions = {
**self.config['interfaces']['required'],
**self.config['interfaces']['optional']
}
def run_tests(self):
for project in self.projects + self.shared_projects:
project.run_tests()
def count_cells(self):
total_cells = 0
for project in self.projects + self.shared_projects:
cells = project.count_cells()
logging.info("%40s %10d cells" % (project.title, cells))
total_cells += cells
logging.info("total standard cells = %s" % total_cells)
# TODO refactor so project konws how to copy gds and lef, then do the same as rtl, gl, test etc.
def copy_all_gds(self):
macros_dir = os.path.join(self.config['caravel']['root'], 'openlane', 'user_project_wrapper', 'macros', 'lef')
lef_dir = os.path.join(self.config['caravel']['root'], 'lef')
gds_dir = os.path.join(self.config['caravel']['root'], 'gds')
for project in self.projects + self.shared_projects:
# gds
src = os.path.join(project.directory, project.gds_filename)
dst = os.path.join(gds_dir, os.path.basename(project.gds_filename))
logging.info("copying %s to %s" % (src, dst))
shutil.copyfile(src, dst)
# lef
src = os.path.join(project.directory, project.lef_filename)
dst = os.path.join(lef_dir, os.path.basename(project.lef_filename))
logging.info("copying %s to %s" % (src, dst))
shutil.copyfile(src, dst)
# gl
project.copy_gl()
# spef, sdc, sdc files
for optional_file_type in ["spef", "sdc", "sdf"]:
dst_dir = os.path.join(self.config['caravel']['root'], optional_file_type)
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
filename = project.get_optional_file(optional_file_type)
if filename is not None:
src = filename
dst = os.path.join(self.config['caravel']['root'], optional_file_type, os.path.basename(filename))
logging.info("copying optional file to %s" % dst)
shutil.copyfile(src, dst)
def copy_all_project_files_to_caravel(self):
### copy out rtl ###
for project in self.projects + self.shared_projects:
project.copy_project_files_to_caravel()
def annotate_image(self):
final_gds_file = os.path.join(self.config['caravel']['root'], 'gds', 'user_project_wrapper.gds.gz')
# dump a 2000x2000 image with klayout to pics/multi_macro.png, check the dump_pic.rb file
cmd = "klayout -l caravel.lyp %s -r dump_pic.rb -c klayoutrc" % final_gds_file
logging.info(cmd)
os.system(cmd)
image_file = os.path.join('pics', 'multi_macro.png')
from PIL import Image, ImageFont, ImageDraw
font_author = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 27)
font_title = ImageFont.truetype("/usr/share/fonts/dejavu/DejaVuSans.ttf", 22)
img = Image.open(image_file)
draw = ImageDraw.Draw(img)
px_per_um = self.config['docs']['px_per_um']
macro_border = self.config['docs']['macro_border']
user_width = self.width * px_per_um
user_height = self.height * px_per_um
x_offset = (2000 - user_width) / 2
y_offset = (2000 - user_height) / 2
logging.info("annotating image")
for project in self.projects + self.shared_projects:
logging.info(project)
macro_x, macro_y, orient = project.get_macro_pos()
x = x_offset + macro_x * px_per_um - macro_border
y = 2000 - (y_offset + macro_y * px_per_um - macro_border) # flip, gds is bottom left 0,0, png is top left 0,0
# takes a while
macro_w, macro_h = project.get_gds_size()
macro_w = macro_w * px_per_um + 2*macro_border
macro_h = macro_h * px_per_um + 2*macro_border
draw.text((x,y-macro_h-70), project.author, (0,0,0), font=font_author)
draw.text((x,y-macro_h-40), project.title, (0,0,0), font=font_title)
draw.line((x, y , x + macro_w, y ), fill=(0,0,0), width=2)
draw.line((x + macro_w, y , x + macro_w, y - macro_h), fill=(0,0,0), width=2)
draw.line((x + macro_w, y - macro_h, x , y - macro_h), fill=(0,0,0), width=2)
draw.line((x , y - macro_h, x , y ), fill=(0,0,0), width=2)
annotated_image_file = os.path.join('pics', 'multi_macro_annotated.png')
img.save(annotated_image_file)
def get_macro_pos(self):
for project in self.projects + self.shared_projects:
logging.info(project.get_macro_pos())
def get_macro_pos_from_caravel(self):
for project in self.projects + self.shared_projects:
logging.info(project.get_macro_pos_from_caravel())
def create_openlane_config(self):
self.generate_macro_cfg()
# self.generate_extra_lef_gds_tcl()
### generate user wrapper verilog and include files ###
user_project_wrapper_path = os.path.join(self.config['caravel']['rtl_dir'], "user_project_wrapper.v")
user_project_includes_path = os.path.join(self.config['caravel']['rtl_dir'], "user_project_includes.v")
caravel_includes_path = os.path.join(self.config['caravel']['includes_dir'], "includes.rtl.caravel_user_project")
# obstruction_path = os.path.join(self.config['caravel']['root'], 'openlane', 'user_project_wrapper', "obstruction.tcl")
# macro_power_path = os.path.join(self.config['caravel']['root'], 'openlane', 'user_project_wrapper', "macro_power.tcl")
generate_openlane_files(
self.projects,
self.shared_projects,
self.interface_definitions,
user_project_wrapper_path,
user_project_includes_path,
caravel_includes_path,
# obstruction_path,
# macro_power_path,
self.args.openram,
self.args.gate_level,
self.config
)
# create the json config file
dst = os.path.join(self.config['caravel']['root'], 'openlane', 'user_project_wrapper', 'config.json')
config = json_config(self.projects)
config.write_config(dst)
def generate_macro_cfg(self):
macro_inst_file = os.path.join(self.config['caravel']['root'], 'openlane', 'user_project_wrapper', 'macro.cfg')
logging.info("overwriting macros.cfg: %s" % macro_inst_file)
with open(macro_inst_file, "w") as f:
for project in self.projects + self.shared_projects:
x, y, orient = project.get_macro_pos()
f.write("%s %.2f %.2f %s\n" % (project.instance_name, x, y, orient))
# want to put macros in top level dirs, so now need to generate a bit of tcl that adds the extra lef/def
def generate_extra_lef_gds_tcl(self):
extra_lef_gds_file = os.path.join(self.config['caravel']['root'], 'openlane', 'user_project_wrapper', 'extra_lef_gds.tcl')
logging.info("creating extra_lef_gds tcl: %s" % extra_lef_gds_file)
with open(extra_lef_gds_file, "w") as f:
f.write('set ::env(EXTRA_LEFS) "')
for project in self.projects + self.shared_projects:
f.write("\\\n $script_dir/../../lef/%s " % os.path.basename(project.lef_filename))
f.write('"\n')
f.write('set ::env(EXTRA_GDS_FILES) "')
for project in self.projects + self.shared_projects:
f.write("\\\n $script_dir/../../gds/%s " % os.path.basename(project.gds_filename))
f.write('"\n')
"""
* generate an index.md with a section for each project
- title, author, description, link, picture
* could also create the info.yaml file for efabless
* tile all images for final image
"""
def generate_docs(self):
git_sha = get_git_sha(".")
fh = open("index.md", 'w')
fh.write("""
[](https://github.com/mattvenn/zero_to_asic_mpw8/actions/workflows/multi_tool.yaml)
# Zero to ASIC Group submission MPW8
This ASIC was designed by members of the [Zero to ASIC course](https://zerotoasiccourse.com).
This submission was configured and built by the [multi project tools](https://github.com/mattvenn/multi_project_tools) at commit [%s](https://github.com/mattvenn/multi_project_tools/commit/%s).
The configuration files are [projects.yaml](projects.yaml) & [local.yaml](local.yaml). See the CI for how the build works.
# clone all repos, and include support for shared OpenRAM
./multi_tool.py --clone-repos --clone-shared-repos --create-openlane-config --copy-gds --copy-project --openram
# run all the tests
./multi_tool.py --test-all --force-delete
# build user project wrapper submission
cd $CARAVEL_ROOT; make user_project_wrapper
# create docs
./multi_tool.py --generate-doc --annotate-image
# Project Index\n\n""" % (git_sha, git_sha))
try_mkdir(self.config["docs"]["pic_dir"], self.args.force_delete)
for project in self.projects:
conf = project.config["project"]
# copy pic
pic_src = os.path.join(project.directory, conf["picture"])
pic_dst = os.path.join(self.config["docs"]["pic_dir"], os.path.basename(conf["picture"]))
shutil.copyfile(pic_src, pic_dst)
fh.write("## %s\n\n" % conf["title"])
fh.write("* Author: %s\n" % conf["author"])
fh.write("* Github: %s\n" % project.repo)
fh.write("* commit: %s\n" % project.gitsha)
fh.write("* Description: %s\n\n" % conf["description"])
fh.write("\n\n" % (conf["title"], pic_dst))
logging.info("wrote index.md")
def prove_all_tristate(self):
sby_file = generate_sby_file(self.projects, self.shared_projects)
logging.info("generated sby file")
with open(os.path.join(self.config['caravel']['root'], "tribuf.sby"), 'w') as fh:
fh.write(sby_file)
try:
sby_cmd = self.config['tools']['sby']
except KeyError:
sby_cmd = 'sby'
cwd = self.config['caravel']['root']
cmd = [sby_cmd, "-f", "tribuf.sby"]
logging.info("attempting to run %s in %s" % (cmd, cwd))
try:
subprocess.run(cmd, cwd=cwd, check=True)
except subprocess.CalledProcessError as e:
logging.error(e)
exit(1)
logging.info("tribuf proof pass")