@@ -40,9 +40,10 @@ void VecLayoutFileIO::makeDir(std::string dir)
}
}
bool VecLayoutFileIO::saveJson()
bool VecLayoutFileIO::saveJson(bool batch_mode )
{
LOG_INFO << "Vectorization save json start... dir = " << _path;
LOG_INFO << "Batch mode: " << (batch_mode ? "enabled (multiple items per file)" : "disabled (single item per file)");
makeDir(_path);
@@ -56,38 +57,35 @@ bool VecLayoutFileIO::saveJson()
saveJsonInstances();
/// save graph
saveJsonNets();
saveJsonNets(batch_mode );
/// save patch
saveJsonPatchs();
saveJsonPatchs(batch_mode );
LOG_INFO << "Vectorization save json end... dir = " << _path;
return true;
}
bool VecLayoutFileIO::saveJsonNets()
bool VecLayoutFileIO::saveJsonNets(bool batch_mode )
{
ieda::Stats stats;
LOG_INFO << "Vectorization save json net start...";
makeDir(_path + "/nets/");
auto& net_map = _layout->get_graph().get_net_map();
const int BATCH_SIZE = 1500; // 可根据系统性能调整批量大小
const int BATCH_SIZE = 1500; // adjustable batch size based on system performance
const int num_threads = omp_get_max_threads();
const int NETS_PER_FILE = 1000; // 每个文件存储的net数量
const int NETS_PER_FILE = 1000; // each file stores this many nets
// 预先将map的键值对复制到vector中,避免O(N^2)的迭代复杂度
// Pre-copy the map's key-value pairs into a vector to avoid O(N²) iteration complexity
std::vector<std::pair<int, VecNet*>> net_vec;
net_vec.reserve(net_map.size());
for (auto& [net_id, vec_net] : net_map) {
net_vec.emplace_back(net_id, &vec_net);
}
// 计算需要的文件数量
int num_files = (net_vec.size() + NETS_PER_FILE - 1) / NETS_PER_FILE;
// 用于收集所有线程生成的JSON数据
// Collect JSON data generated by all threads
std::vector<std::vector<std::pair<int, json>>> thread_batches(num_threads);
int total = 0;
@@ -95,11 +93,10 @@ bool VecLayoutFileIO::saveJsonNets()
{
int thread_id = omp_get_thread_num();
auto& local_batch = thread_batches[thread_id];
local_batch.reserve(BATCH_SIZE + 100); // 预分配空间
local_batch.reserve(BATCH_SIZE + 100); // preallocate space
#pragma omp for schedule(dynamic, 100) reduction(+ : total)
for (int i = 0; i < (int) net_vec.size(); ++i) {
// 直接O(1)访问vector元素,而不是O(i)的std::advance
const auto& [net_id, vec_net_ptr] = net_vec[i];
auto& vec_net = *vec_net_ptr;
auto* idb_net = dmInst->get_idb_design()->get_net_list()->get_net_list()[net_id];
@@ -265,7 +262,7 @@ bool VecLayoutFileIO::saveJsonNets()
json_net["routing_graph"] = json_routing_graph;
}
// 将结果添加到本地批次中,存储net_id和对应的json数据
// Add the result to the local batch, storing net_id and corresponding json data
local_batch.emplace_back(net_id, std::move(json_net));
if (i % 1000 == 0) {
@@ -277,10 +274,10 @@ bool VecLayoutFileIO::saveJsonNets()
}
}
// 并行区域结束后,合并所有线程的结果
// After the parallel region, merge the results of all threads
LOG_INFO << "JSON generation completed, merging results...";
// 创建一个映射,将net_id映射到对应的json数据
// Create a mapping from net_id to corresponding json data
std::map<int, json> all_nets;
for (const auto& batch : thread_batches) {
for (const auto& [net_id, json_data] : batch) {
@@ -288,41 +285,67 @@ bool VecLayoutFileIO::saveJsonNets()
}
}
// 批量写入文件
LOG_INFO << "Starting batch file writing...";
// 计算需要写入的文件数量
int total_files = (all_nets.size() + NETS_PER_FILE - 1) / NETS_PER_FILE;
if (batch_mode) {
// batch file mode: multiple nets per file
LOG_INFO << "Starting batch file writing...";
int total_files = (all_nets.size() + NETS_PER_FILE - 1) / NETS_PER_FILE;
#pragma omp parallel for schedule(dynamic, 1) num_threads(std::min(num_threads, 8))
for (int file_idx = 0; file_idx < total_files; ++file_idx) {
// 计算当前文件包含的网络范围
int start_net_idx = file_idx * NETS_PER_FILE;
int end_net_idx = std::min((file_idx + 1) * NETS_PER_FILE - 1, (int) all_nets.size() - 1);
for (int file_idx = 0; file_idx < total_files; ++file_idx) {
int start_net_idx = file_idx * NETS_PER_FILE;
int end_net_idx = std::min((file_idx + 1) * NETS_PER_FILE - 1, (int) all_nets.size() - 1);
// 创建文件名格式: net_START_END .json
std::string filename = "net_" + std::to_string(start_net_idx) + "_" + std::to_string(end_net_idx) + ".json";
std::string full_path = _path + "/nets/" + filename;
// create file name format: net_start_end .json
std::string filename = "net_" + std::to_string(start_net_idx) + "_" + std::to_string(end_net_idx) + ".json";
std::string full_path = _path + "/nets/" + filename;
// 创建一个包含当前批次网络的数组
json batch_json = json::array();
json batch_json = json::array();
// 找到这个范围内的所有网络
auto it = all_nets.begin();
std::advance(it, start_net_idx);
auto it = all_nets.begin();
std::advance(it, start_net_idx);
for (int i = start_net_idx; i <= end_net_idx && it != all_nets.end(); ++i, ++it) {
batch_json.push_back(it->second);
}
for (int i = start_net_idx; i <= end_net_idx && it != all_nets.end(); ++i, ++it) {
batch_json.push_back(it->second);
}
std::ofstream file_stream(full_path);
// file_stream << std::setw(4) << batch_json;
file_stream << batch_json;
file_stream.close();
std::ofstream file_stream(full_path);
file_stream << batch_json;
file_stream.close();
#pragma omp critical(log)
{
LOG_INFO << "Writing files: " << (file_idx + 1) * NETS_PER_FILE << " / " << all_nets.size();
{
LOG_INFO << "Writing files: " << (file_idx + 1) * NETS_PER_FILE << " / " << all_nets.size();
}
}
} else {
// individual file mode: one net per file
LOG_INFO << "Starting individual file writing...";
// Convert map to vector for OpenMP parallel processing
std::vector<std::pair<int, json>> net_list;
for (const auto& [net_id, json_data] : all_nets) {
net_list.emplace_back(net_id, json_data);
}
#pragma omp parallel for schedule(dynamic, 10) num_threads(std::min(num_threads, 8))
for (int i = 0; i < (int)net_list.size(); ++i) {
const auto& [net_id, json_data] = net_list[i];
// create file name format: net_id.json
std::string filename = "net_" + std::to_string(net_id) + ".json";
std::string full_path = _path + "/nets/" + filename;
std::ofstream file_stream(full_path);
file_stream << std::setw(4) << json_data;
file_stream.close();
if ((i + 1) % 1000 == 0 || i == (int)net_list.size() - 1) {
#pragma omp critical(log)
{
LOG_INFO << "Writing individual files: " << (i + 1) << " / " << net_list.size();
}
}
}
}
@@ -335,7 +358,7 @@ bool VecLayoutFileIO::saveJsonNets()
return true;
}
bool VecLayoutFileIO::saveJsonPatchs()
bool VecLayoutFileIO::saveJsonPatchs(bool batch_mode )
{
ieda::Stats stats;
LOG_INFO << "Vectorization save json patchs start...";
@@ -346,28 +369,23 @@ bool VecLayoutFileIO::saveJsonPatchs()
}
auto& patchs = _patch_grid->get_patchs();
const int BATCH_SIZE = 1500; // 可根据系统性能调整批量大小
const int BATCH_SIZE = 1500; // adjustable batch size based on system performance
const int num_threads = omp_get_max_threads();
const int PATCHS_PER_FILE = 1000; // 每个文件存储的patch数量
const int PATCHS_PER_FILE = 1000; // each file stores this many patchs
// 预先将map的键值对复制到vector中,避免O(N²)的迭代复杂度
std::vector<std::pair<int, VecPatch*>> patch_vec;
patch_vec.reserve(patchs.size());
for (auto& [patch_id, patch] : patchs) {
patch_vec.emplace_back(patch_id, &patch);
}
// 计算需要的文件数量
int num_files = (patch_vec.size() + PATCHS_PER_FILE - 1) / PATCHS_PER_FILE;
// 用于收集所有线程生成的JSON数据
std::vector<std::vector<std::pair<int, json>>> thread_batches(num_threads);
#pragma omp parallel
{
int thread_id = omp_get_thread_num();
auto& local_batch = thread_batches[thread_id];
local_batch.reserve(BATCH_SIZE + 100); // 预分配空间
local_batch.reserve(BATCH_SIZE + 100);
#pragma omp for schedule(dynamic, 100)
for (int i = 0; i < (int) patch_vec.size(); ++i) {
@@ -509,7 +527,7 @@ bool VecLayoutFileIO::saveJsonPatchs()
json_patch["patch_layer"] = json_layers;
}
// 将结果添加到本地批次中,存储patch_id和对应的json数据
// Add the result to the local batch, storing patch_id and corresponding json data
local_batch.emplace_back(patch_id, std::move(json_patch));
if (i % 1000 == 0) {
@@ -521,10 +539,8 @@ bool VecLayoutFileIO::saveJsonPatchs()
}
}
// 并行区域结束后,合并所有线程的结果
LOG_INFO << "JSON generation completed, merging results...";
// 创建一个映射,将patch_id映射到对应的json数据
std::map<int, json> all_patches;
for (const auto& batch : thread_batches) {
for (const auto& [patch_id, json_data] : batch) {
@@ -532,40 +548,64 @@ bool VecLayoutFileIO::saveJsonPatchs()
}
}
// 批量写入文件
LOG_INFO << "Starting batch file writing...";
// 计算需要写入的文件数量
int total_files = (all_patches.size() + PATCHS_PER_FILE - 1) / PATCHS_PER_FILE;
if (batch_mode) {
LOG_INFO << "Starting batch file writing...";
int total_files = (all_patches.size() + PATCHS_PER_FILE - 1) / PATCHS_PER_FILE;
#pragma omp parallel for schedule(dynamic, 1) num_threads(std::min(num_threads, 8))
for (int file_idx = 0; file_idx < total_files; ++file_idx) {
// 计算当前文件包含的patch范围
int start_patch_idx = file_idx * PATCHS_PER_FILE;
int end_patch_idx = std::min((file_idx + 1) * PATCHS_PER_FILE - 1, (int) all_patches.size() - 1);
for (int file_idx = 0; file_idx < total_files; ++file_idx) {
int start_patch_idx = file_idx * PATCHS_PER_FILE;
int end_patch_idx = std::min((file_idx + 1) * PATCHS_PER_FILE - 1, (int) all_patches.size() - 1);
// create a file name format: patch_start_end.json
std::string filename = "patch_" + std::to_string(start_patch_idx) + "_" + std::to_string(end_patch_idx) + ".json";
std::string full_path = _path + "/patchs/" + filename;
// 创建文件名格式: patch_START_END.json
std::string filename = "patch_" + std::to_string(start_patch_idx) + "_" + std::to_string(end_patch_idx) + ".json";
std::string full_path = _path + "/patchs/" + filename;
json batch_json = json::array();
// 创建一个包含当前批次patch的数组
json batch_json = json::array();
auto it = all_patches.begin();
std::advance(it, start_patch_idx);
for (int i = start_patch_idx; i <= end_patch_idx && it != all_patches.end(); ++i, ++it) {
batch_json.push_back(it->second);
}
// 找到这个范围内的所有patch
auto it = all_patches.begin();
std::advance(it, start_patch_idx);
std::ofstream file_stream(full_path);
file_stream << batch_json ;
file_stream.close( );
for (int i = start_patch_idx; i <= end_patch_idx && it != all_patches.end(); ++i, ++it) {
batch_json.push_back(it->second);
#pragma omp critical(log)
{
LOG_INFO << "Writing files: " << (file_idx + 1) * PATCHS_PER_FILE << " / " << all_patches.size();
}
}
} else {
LOG_INFO << "Starting individual file writing...";
std::vector<std::pair<int, json>> patch_list;
for (const auto& [patch_id, json_data] : all_patches) {
patch_list.emplace_back(patch_id, json_data);
}
std::ofstream file_stream(full_path);
file_stream << batch_json;
file_stream.close();
#pragma omp parallel for schedule(dynamic, 10) num_threads(std::min(num_threads, 8))
for (int i = 0; i < (int) patch_list.size(); ++i) {
const auto& [patch_id, json_data] = patch_list[i];
// create file name format: patch_id.json
std::string filename = "patch_" + std::to_string(patch_id) + ".json";
std::string full_path = _path + "/patchs/" + filename;
std::ofstream file_stream(full_path);
file_stream << std::setw(4) << json_data;
file_stream.close();
#pragma omp critical(log)
{
LOG_INFO << "Writing files: " << (file_idx + 1) * PATCHS_PER_FILE << " / " << all_patches.size();
{
if ((i + 1) % 100 == 0 || i == (int) patch_list.size() - 1) {
LOG_INFO << "Writing individual files: " << (i + 1) << " / " << patch_list.size();
}
}
}
}
@@ -950,8 +990,11 @@ bool VecLayoutFileIO::readJsonNetsPattern()
}
std::string layer_metal = _layout->findLayerName(layer_index_start);
auto* idb_layer_metal = idb_layers->find_layer(layer_metal);
if (layer_metal == "") {
continue;
}
auto* idb_layer_metal = idb_layers->find_layer(layer_metal);
auto* idb_segment = idb_wire->add_segment();
idb_segment->set_layer(idb_layer_metal);
@@ -968,8 +1011,12 @@ bool VecLayoutFileIO::readJsonNetsPattern()
for (auto layer_order = bottom_order; layer_order <= top_order; layer_order += 2) {
std::string bottom_layer_name = _layout->findLayerName(layer_order);
auto* bottom_layer = idb_layers->find_layer(bottom_layer_name);
std::string top_layer_name = _layout->findLayerName(layer_order + 2);
if (bottom_layer_name == "" || top_layer_name == "") {
continue;
}
auto* bottom_layer = idb_layers->find_layer(bottom_layer_name);
auto* top_layer = idb_layers->find_layer(top_layer_name);
auto* idb_segment = idb_wire->add_segment();