Tidyverse 🪐to Polars 🐻‍❄️: My Notes

df |>
  filter(age >= 30) |>
  select(1:3, starts_with("performance"), starts_with("salary")) |> 
  summarize(across(.cols = where(is.numeric), .fns = mean, .names = "mean_{.col}"))

## # A tibble: 1 × 5
##   mean_age mean_performance_rating mean_performance_reviews_count mean_salary
##                                                          
## 1     46.6                     4.4                            3.4       40000
## # ℹ 1 more variable: mean_salary_increase_percentage 

df \
    .filter(pl.col('age') >= 30) \
    .select(df.columns[0:4]+['^performance.*$','^salary.*$']) \
    .select(pl.col(pl.Int64).mean().name.prefix('mean_'))

For some reason, for the regex above, I have to use ^ and $ sandwiched to return those column nams that I want to include. bizzare.

df \
    .with_columns(
        pl.concat_str(
            pl.col(pl.String), separator=" "
        ).alias('combination_of_character')
    ) \
    .select(pl.col('combination_of_character'))

Test 2

• make a new column called age_salary
• glue column age and salary together with - between
• select columns name and age_salary

df |> 
  mutate(age_salary = paste0(age, "-", salary)) |>
  select(name, age_salary)

## # A tibble: 6 × 2
##   name    age_salary
##           
## 1 Alice   30-50000  
## 2 Bob     25-45000  
## 3 Charlie 35-60000  
## 4 Ken     50-20000  
## 5 Steven  60-40000  
## 6 Carlos  58-30000

df \
    .with_columns(
        age_salary=pl.format('{}-{}',pl.col('age'),pl.col('salary'))
    ) \
    .select(pl.col('name','age_salary'))

If it’s just 1 column, you can use this format age_salary= to name the column, otherwise you’d have to use alias to name it if there are multple columns

• create a new column area_code_and_salary
• paste street number (extract it from address) with a space and the the column salary
• select area_code_and_salary

df |>
  mutate(area_code_and_salary = paste0(str_extract(address, "\\d{0,5}"), " ", salary)) |>
  select(area_code_and_salary)

## # A tibble: 6 × 1
##   area_code_and_salary
##                  
## 1 123 50000           
## 2 123 45000           
## 3 456 60000           
## 4 49494 20000         
## 5 1010 40000          
## 6 666 30000

df \
    .select(
        pl.concat_str(
            pl.col('address').str.extract(r'^(\d{0,5})'),
            pl.lit(" "),
            pl.col('salary')
        ).alias('area_code_and_salary')
    )

Have to use pl.lit(' ') for any constant string

df |>
  mutate(familiarity = case_when(
    str_detect(address, "OH") ~ "local",
    str_detect(address, "NY") ~ "foodie",
    TRUE ~ "elsewhere"
  )) 

## # A tibble: 6 × 17
##   name      age city      address phone_number email salary department hire_date
##                                    
## 1 Alice      30 New York  123 Ma… 123-456-7890 alic…  50000 Engineeri… 2010-01-…
## 2 Bob        25 San Fran… 123 Ma… 987-654-3210 bob@…  45000 Marketing  2012-05-…
## 3 Charlie    35 Tokyo     456-78… 098-765-4332 char…  60000 Finance    2015-10-…
## 4 Ken        50 Toronto   49494 … 111-232-4141 ken@…  20000 Marketing  2010-04-…
## 5 Steven     60 Lima      1010 G… 505-402-6060 step…  40000 Marketing  2009-10-…
## 6 Carlos     58 Cleveland 666 He… 909-435-1000 carl…  30000 Finance    2005-11-…
## # ℹ 8 more variables: status , salary_increase_percentage ,
## #   years_of_service , bonus_amount , performance_rating ,
## #   performance_reviews_count , performance_reviews_last_updated ,
## #   familiarity 

df \
    .with_columns([  
        pl.when(pl.col('address').str.contains('OH'))
        .then(pl.lit('local'))
        .when(pl.col('address').str.contains('NY'))
        .then(pl.lit('foodie'))
        .otherwise(pl.lit('elsewhere'))
        .alias('familiarity')
    ])

Test 2

• convert name data to lowercase
• create new column called email_name and extract email before the @
• select columns that starts with name or end with name
• create a new column called same?
• if name and email_name is the same, then return yes
• otherwise return no

df |>
  mutate(
    name = tolower(name),
    email_name = str_extract(email, "^([\\d\\w]+)@", group = 1)
  ) |>
  select(starts_with("name") | ends_with("name")) |>
  mutate(`same?` = case_when(
    name == email_name ~ "yes",
    TRUE ~ "no"))

## # A tibble: 6 × 3
##   name    email_name   `same?`
##                
## 1 alice   alice        yes    
## 2 bob     bob          yes    
## 3 charlie charlie      yes    
## 4 ken     ken          yes    
## 5 steven  stephencurry no     
## 6 carlos  carlos       yes

df \
    .with_columns(
        [
        pl.col('name').str.to_lowercase(),    
        pl.col('email').str.extract(r'^([\d\w]+)@', group_index = 1)
        .alias('email_name')
        ]
    ) \
    .select([
        pl.col('^name|.*name$'),
        pl.when(
            pl.col('name') == pl.col('email_name')).then(pl.lit('yes'))
            .otherwise(pl.lit('no'))
            .alias('same?')
    ]
        )

Learnt that apparently we cannot use look forward or backward in polars. Such as .*(?=@) to capture the email_name

df |>
  group_by(department) |>
  summarize(
    name = name,
    salary_shift = case_when(
      n() == 1 ~ salary,
      TRUE ~ lead(salary)
    )
  ) |>
 fill(salary_shift, .direction = "down")

## Warning: Returning more (or less) than 1 row per `summarise()` group was deprecated in
## dplyr 1.1.0.
## ℹ Please use `reframe()` instead.
## ℹ When switching from `summarise()` to `reframe()`, remember that `reframe()`
##   always returns an ungrouped data frame and adjust accordingly.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.

## `summarise()` has grouped output by 'department'. You can override using the
## `.groups` argument.

## # A tibble: 6 × 3
## # Groups:   department [3]
##   department  name    salary_shift
##                    
## 1 Engineering Alice          50000
## 2 Finance     Charlie        30000
## 3 Finance     Carlos         30000
## 4 Marketing   Bob            20000
## 5 Marketing   Ken            40000
## 6 Marketing   Steven         40000

df \
.group_by('department') \
.agg(
    pl.col('name'),
    pl.when(pl.col('salary').len()==1).then(pl.col('salary'))
    .otherwise(pl.col('salary').shift(-1))
    .alias('salary_shift')) \
.explode('name','salary_shift') \
.with_columns(
    pl.col('salary_shift').forward_fill())

Apparently polars would turn the column into a nested dataframe (list) when grouped and can’t do fill when it’s in list? will have to unnest by explode before fill can be used. Unless of coure if you merge the fill in the same line when shifting, such as

df \
.group_by('department') \
.agg(
    pl.col('name'),
    pl.when(pl.col('salary').len()==1).then(pl.col('salary'))
    .otherwise(pl.col('salary').shift(-1))
    .forward_fill() 
    .alias('salary_shift'))

Yes! I believe pl.col, pl.select, pl.filter all take a list of conditions. First create a list of columns you want to unnest, then use pl.col to select them.

dt = [pl.List(pl.Int64),pl.List(pl.String)]

df \
    .group_by('department', maintain_order=True) \
    .agg(  
        pl.col('name'),
        pl.col('salary_increase_percentage'),
        salary_shift = pl.when(pl.col('salary').count() == 1)  
            .then(pl.col('salary'))
            .otherwise(pl.col('salary').shift(-1))
    ) \
    .explode(pl.col(dt)) \
    .with_columns(
        pl.col('salary_shift').forward_fill()
    ) \
    .with_columns(
        new_raise = pl.col('salary_shift') * (1+pl.col('salary_increase_percentage')/100)
    )

df_dept <- tibble(
  department = c("Engineering", "Marketing", "Finance"),
  dept_id = c(30, 25, 20)
)

df_dept = pl.DataFrame({
    "department": ["Engineering", "Marketing", "Finance"],
    "dept_id": [30, 25, 20]
})

• left_join df with df_dept
• create new column employee_id by pasting:
  • {dept_id}-{random 7 digit numbers}

df |>
  left_join(df_dept, by = "department") |>
  select(name, dept_id) |>
  mutate(employee_id = map_chr(dept_id, ~paste0(.x, "-", sample(1000000:9999999, 1))))

## # A tibble: 6 × 3
##   name    dept_id employee_id
##               
## 1 Alice        30 30-1694470 
## 2 Bob          25 25-1696036 
## 3 Charlie      20 20-4463080 
## 4 Ken          25 25-6942432 
## 5 Steven       25 25-3012223 
## 6 Carlos       20 20-8705991

import random

df \
    .join(df_dept, on="department") \
    .select(['name','dept_id']) \
    .with_columns(
        employee_id = pl.format(
          '{}-{}',
          'dept_id',
          pl.Series([
            random.randint(100000, 999999) for _ in range(len(df))
            ])
            )
    )

there is a function called map_elements in polars but the documentation stated that it’s inefficient, essentially using a for loop. I’m not entirely certain if list comprehension above is any more efficient. Another probably more efficent way of doing this is 2 separate process. The random number generation on another dataframe, then merge it.

df |>
  select(name, department) |>
  pivot_wider(id_cols = "name", names_from = "department", values_from = "department", values_fill = 0, values_fn = length, names_prefix = "department_")

## # A tibble: 6 × 4
##   name    department_Engineering department_Marketing department_Finance
##                                                     
## 1 Alice                        1                    0                  0
## 2 Bob                          0                    1                  0
## 3 Charlie                      0                    0                  1
## 4 Ken                          0                    1                  0
## 5 Steven                       0                    1                  0
## 6 Carlos                       0                    0                  1

df \
    .select(['name','department']) \
    .to_dummies(columns = 'department') 

df \
    .select(['name','address']) \
    .with_columns(
        state = pl.col('address').str.extract(r'([A-Z]{2})$')
    ) \
    .select('name','state') \
    .pivot(on = 'state', index = 'name', values='state', aggregate_function='len') \
    .with_columns(
        pl.col(pl.UInt32).fill_null(0)
    ) 

Essentially, pivot_wider is Polars’ pivot. Whereas pivot_longer is Polars’ unpivot