In benjaminguinaudeau/bashR:

knitr::opts_chunk$set(echo = TRUE)

pacman::p_load(tidyverse, dockeR, bashR, reticulate, magick)
reticulate::use_python("/Users/benjaminguinaudeau/anaconda3/bin/python")

mouse <- import("pyautogui")

transform_coord <- function(x = NULL, y = NULL, x_sel = NULL, y_sel = NULL){
  if(!is.null(x)){return(x)}
  if(!is.null(x_sel)){return(x_sel)}
  if(!is.null(y)){return((y - 120))}
  if(!is.null(y_sel)){return((y_sel + 120))}
}

random_scroll <- function(){
  1:10 %>%
    map(~{
      down <- sample(-1:-10, 1) %>% glimpse 
      up <- (sample(1:abs(as.integer(down)), 1) + runif(1, 0, 1))/2
      doc_scroll((down - runif(1, 0, 1)))
      if(runif(1,0,1) < .2){
        wait(rpois(1, 5)*.05, 0.1, verbose = T)
      }
      # wait(rpois(1, 10)*.2, 0.1, verbose = T)
      if(runif(1,0,1) < .10){
        doc_scroll(up)
        print("up")
      }
    })
}

elems <-  chrome %>% dockeR::elements(".article__title-label")
selenium_pos <-elems %>%
  simule_map(1)
map(~{
  tmp <- as_tibble(.x$getElementLocation()) %>%
    glimpse
  Sys.sleep(1)
  doc_mouse_moveTo(transform_coord(x_sel = tmp$x), transform_coord(y_sel = tmp$y))
  doc_mouse_click(button = "right")
  Sys.sleep(2)
  doc_mouse_click(button = "right")
})
Sys.sleep(2)


tibble(
  x = c(76, 283, 458, 664, 871),
  x_sel = c(16, 215, 411, 607, 803),
  y = c(370, 365, 370, 364, 358)
) %>%
  mutate(diff = x_sel / x)

chrome <- chrome_init("chrome")
view_container("chrome")
chrome$closeall()
chrome$open()

chrome %>% go("http://twitter.com")
chrome$maxWindowSize()
wait()

doc_mouse_click(sample(753:960, 1), sample(152:191, 1), button = "left")
doc_mouse_type("Ben_Guinaudeau", interval = .15)
wait()
doc_mouse_click(sample(975:1180, 1), sample(152:191, 1), button = "left")
doc_mouse_type("guxfir-8qyfso-zomsYk", interval = .15)
wait()
doc_mouse_click(sample(1212:1246, 1), sample(152:191, 1), button = "left")

# dockeR::doc_copy("chrome", from_cont = "comment.png", to = "/Users/benjaminguinaudeau/comment.png")

doc_screenshot(file = "'/screeen.png'")
doc_copy("chrome", from_cont = "/screeen.png", to = "/Users/benjaminguinaudeau/screen.png")

pos <- mouse$locate("/Users/benjaminguinaudeau/comment.png", "/Users/benjaminguinaudeau/screen.png") %>%
  as.character

x <- stringr::str_extract(pos, "(?<=left=)\\d+") %>% as.numeric()
y <- stringr::str_extract(pos, "(?<=top=)\\d+") %>% as.numeric()
width <- stringr::str_extract(pos, "(?<=width=)\\d+") %>% as.numeric()
height <- stringr::str_extract(pos, "(?<=height=)\\d+") %>% as.numeric()

wait()
doc_mouse_click(sample(x:(x + width), 1), sample(y:(y + height), 1), button = "left")

doc_scroll(-x)

random_scroll()

like <- chrome %>%
  elements(".r-4qtqp9.r-yyyyoo.r-1xvli5t.r-dnmrzs.r-bnwqim.r-1plcrui.r-lrvibr.r-1hdv0qi")

like %>%
  map(~{
    tmp <- as_tibble(.x$getElementLocation()) %>%
      glimpse
    Sys.sleep(1)
    doc_mouse_moveTo(transform_coord(x_sel = tmp$x), transform_coord(y_sel = tmp$y))
    doc_mouse_click(button = "right")
    Sys.sleep(2)
    doc_mouse_click(button = "right")
  })

# doc_screenshot(file = "'comment.png'", region = '(379, 754, 42, 32)')
doc_mouse_position()

doc_locate_on_screen("'comment.png'")

pacman::p_load(jpeg, imager, magick, furrr)
plan(multiprocess)
library(jpeg)
# 
# kern <- matrix(0, ncol = 3, nrow = 3)
# kern[1, 2] <- 0.25
# kern[2, c(1, 3)] <- 0.25
# kern[3, 2] <- 0.25
# kern

screen <- load.image('~/screen.png') %>%
  grayscale %>%
  plot
  as.matrix

comment <- load.image('~/comment.png') %>%
  grayscale %>%
  as.matrix

euc.dist <- function(x1, x2){
  sqrt(sum(
    matrix(x1[,,1]) - matrix(x2[,,1])^ 2,
    matrix(x1[,,2]) - matrix(x2[,,2])^ 2,
    matrix(x1[,,3]) - matrix(x2[,,3]) ^ 2))
}

ccf

x_com <- ncol(comment)
y_com <- nrow(comment)
x_sc <- ncol(screen)
y_sc <- nrow(screen)
x_stop <- x_sc - x_com 
y_stop <- y_sc - y_com 


results <- tibble()
for(x in 1:x_stop){
  if(x %% 100 == 0){message(x)}
  for(y in 1:y_stop){
    sub_x <- x + x_com -1
    sub_y <- y + y_com - 1
    sub <- as.vector(screen[y:sub_y, x:sub_x])
    if(all(sub == as.vector(comment))|
       sum(sub == as.vector(comment)) > (.7*x_com*y_com)){
      results <- bind_rows(results, tibble(x = x, y = y, 
                                           a = sum(sub == as.vector(comment))))
      # message("x = ", x,", y = ", y)
    }
  }
}

results %>%
  ggplot(aes(x =x,y=y, color = a)) + geom_point()

test <- 1:(ncol(screen) - ncol(comment)) %>%
  # simule_map(1)
  future_map(~{
    x <- .x
    1:(nrow(screen) - nrow(comment)) %>%
      # simule_map(1)
      map_dbl(~{

        n_row <- .x:(.x + nrow(comment) - 1)
        n_col <- x:(x + ncol(comment) - 1)
        x2 <- screen[n_row,
                     n_col]
        out <- sum(as.vector(comment) == as.vector(x2))
        return(out)
      })
  }, .progress = T) %>%
  reduce(c) %>%
  matrix(nrow(screen) - nrow(comment), ncol(screen) - ncol(comment))

test %>%
  as_tibble %>%
  mutate(x = 1:n()) %>%
  pivot_longer(-x) %>%
  mutate(y = as.numeric(str_extract(name, "\\d+")), 
         dist = scale(value)) %>%
  ggplot(aes(x, -y, color = value)) + geom_point() + viridis::scale_color_viridis()

x <- which(test == min(test)) %/% ncol(screen)
y <- which(test == min(test)) %% ncol(screen)

doc_mouse_click(x + 125 + ncol(comment)/2, y + 125+ ncol(comment)/2, button = "right")



comment <- image_read('~/comment.png') %>%

  img_blurred <- image_convolve(screen, kern)
img_blurred



screen
comment

benjaminguinaudeau/bashR documentation built on May 9, 2023, 11:46 p.m.