Skip to content

Enable new CI pipeline for standard & distributed tests #1481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jul 27, 2022
Merged

Enable new CI pipeline for standard & distributed tests #1481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
ca4d48c
debug
malachi-constant Jul 25, 2022
4b66a7f
Merge branch 'release-3.0.0' of github.com:awslabs/aws-data-wrangler …
malachi-constant Jul 25, 2022
ca7d76d
Minor - Add autoimport pandas env var to ray init to fix failing pyte…
jaidisido Jul 25, 2022
7ee106c
Merge branch 'autoimport-pandas' of github.com:awslabs/aws-data-wrang…
malachi-constant Jul 26, 2022
ac00aac
Marking distributed tests
malachi-constant Jul 26, 2022
94de53a
Register marker in pyproject
malachi-constant Jul 26, 2022
e758b5d
Adding global mark
malachi-constant Jul 26, 2022
26d6c44
removing unecessary assert for wr.config.distributed
malachi-constant Jul 26, 2022
155d907
setting environment var in codebuild
malachi-constant Jul 27, 2022
5798246
Adding environment variable assembly
malachi-constant Jul 27, 2022
cd3d615
Testing env var with tox
malachi-constant Jul 27, 2022
d49bc83
reverting to comprehenstion of ray runtime env vars
malachi-constant Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions awswrangler/distributed/_distributed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,10 @@ def initialize_ray(
)
os.environ["RAY_ENABLE_MAC_LARGE_OBJECT_STORE"] = "1"

ray_runtime_env_vars = [
"__MODIN_AUTOIMPORT_PANDAS__",
]

ray_init_kwargs = {
"num_cpus": cpu_count or multiprocessing.cpu_count(),
"num_gpus": gpu_count,
Expand All @@ -115,6 +119,9 @@ def initialize_ray(
"object_store_memory": object_store_memory,
"_redis_password": redis_password,
"_memory": object_store_memory,
"runtime_env": {
"env_vars": {var: os.environ.get(var) for var in ray_runtime_env_vars if os.environ.get(var)}
},
}
ray.init(**ray_init_kwargs)

Expand Down
3 changes: 3 additions & 0 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,3 +129,6 @@ warn_unused_ignores = false
log_cli = false
filterwarnings = "ignore::DeprecationWarning"
addopts = "--log-cli-format \"[%(name)s][%(funcName)s] %(message)s\" --verbose --capture=sys"
markers = [
"distributed: tests againsts methods with distributed functionality",
]
3 changes: 3 additions & 0 deletions tests/test_lakeformation.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
import time

import pandas as pd
import pytest

import awswrangler as wr

from ._utils import ensure_data_types, ensure_data_types_csv, get_df, get_df_csv

logging.getLogger("awswrangler").setLevel(logging.DEBUG)

pytestmark = pytest.mark.distributed


def test_lakeformation(path, path2, glue_database, glue_table, glue_table2, use_threads=False):
wr.catalog.delete_table_if_exists(database=glue_database, table=glue_table)
Expand Down
2 changes: 2 additions & 0 deletions tests/test_s3_select.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

logging.getLogger("awswrangler").setLevel(logging.DEBUG)

pytestmark = pytest.mark.distributed


@pytest.mark.parametrize("use_threads", [True, False, 2])
def test_full_table(path, use_threads):
Expand Down