Skip to content
This repository was archived by the owner on Nov 22, 2022. It is now read-only.

Commit 38f65f9

Browse files
hudevenfacebook-github-bot
authored andcommitted
Move MockConfigLoader to OSS (#1324)
Summary: Pull Request resolved: #1324 1. harding coding "/tmp/model.pt" in config caused race condition when multiple tests are running concurrently 2. we patched the output path with "/tmp{RANDOM_HASH}/model.pt" by MockConfigLoader 3. I want to reuse the same patch logic in OSS integration tests Differential Revision: D21084986 fbshipit-source-id: 400f6d43a134e189bb2adb1ae4a13a592df1ba7b
1 parent 9834315 commit 38f65f9

File tree

1 file changed

+61
-0
lines changed

1 file changed

+61
-0
lines changed

pytext/utils/config_utils.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
3+
4+
import json
5+
import os
6+
from importlib import import_module
7+
8+
from pytext.config import PyTextConfig, config_to_json
9+
from pytext.config.config_adapter import upgrade_to_latest
10+
from pytext.config.serialize import pytext_config_from_json
11+
from pytext.utils.file_io import PathManager
12+
13+
14+
class MockConfigLoader:
15+
def __init__(
16+
self,
17+
config_base_path: str,
18+
reset_paths: {str: str} = None,
19+
replace_paths: {str: str} = None,
20+
):
21+
self.config_base_path = config_base_path
22+
self.reset_paths = reset_paths or {}
23+
self.replace_paths = replace_paths or {}
24+
25+
def make_config(self, config_filename, disable_tensorboard=True):
26+
with PathManager.open(
27+
os.path.join(self.config_base_path, config_filename)
28+
) as config_file:
29+
# load config json file into dict
30+
config_dict = json.load(config_file)
31+
if "config" in config_dict:
32+
config_dict = config_dict["config"]
33+
return self.make_config_from_dict(config_dict, disable_tensorboard)
34+
35+
def make_config_from_dict(self, config, disable_tensorboard):
36+
# config is the path module name of the actual PyText config
37+
if isinstance(config, str):
38+
config = config_to_json(PyTextConfig, import_module(config).config)
39+
config = upgrade_to_latest(config)
40+
# Disable TensorBoard for integration tests
41+
if disable_tensorboard:
42+
config["use_tensorboard"] = False
43+
return self.disable_cuda(self.fix_paths(config))
44+
45+
def disable_cuda(self, config):
46+
config["use_cuda_if_available"] = False
47+
if "distributed_world_size" in config:
48+
del config["distributed_world_size"]
49+
return config
50+
51+
def fix_paths(self, config):
52+
if isinstance(config, str):
53+
for src, dest in self.replace_paths.items():
54+
config = config.replace(src, dest)
55+
return config
56+
elif isinstance(config, dict):
57+
return {key: self.fix_paths(value) for key, value in config.items()}
58+
elif isinstance(config, list):
59+
return [self.fix_paths(element) for element in config]
60+
else:
61+
return config

0 commit comments

Comments
 (0)